From 1607b9d86223826a2aa2705a55b833edacbf16cc Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 00:33:08 +0530 Subject: [PATCH 01/93] Add the MCP tool-surface eval harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures how well an agent completes real Plane tasks through an MCP tool surface, so tool-surface decisions rest on measurements instead of estimates — the last consolidation proposal's cost claims were off by 3.4x when measured. 34 tasks run against a live Plane API with fixtures seeded and torn down per task, each graded by a verifier that reads state back through the API rather than reading the agent's prose. Per task it reports pass/fail, tool calls to done, which tools were picked and whether they were optimal, errors, and the final answer. The harness is agent- and surface-agnostic. Five drivers (Codex, Claude Code, Antigravity, opencode, and the Anthropic SDK tool runner) all record real JSON-RPC traffic through a recording proxy, so call counts come off the wire rather than from self-report. Any stdio MCP server can be measured with --server-cmd, which is how competing PR surfaces were compared head to head. evals/README.md is the runbook; evals/DESIGN.md is the rationale. The task catalog keeps per-surface overlays so a surface that genuinely cannot do something is reported as a capability gap rather than a failure. Co-Authored-By: Claude Fable 5 --- .gitignore | 6 + evals/DESIGN.md | 358 ++++ evals/README.md | 176 ++ evals/__init__.py | 1 + evals/cleanup.py | 109 + evals/drivers.py | 1947 ++++++++++++++++++ evals/env.sh | 282 +++ evals/listing.py | 247 +++ evals/mock_flags.py | 120 ++ evals/proxy.py | 587 ++++++ evals/report.py | 645 ++++++ evals/run.py | 1203 +++++++++++ evals/seed.py | 1152 +++++++++++ evals/tasks.py | 2798 ++++++++++++++++++++++++++ pyproject.toml | 4 + tests/test_evals_catalog.py | 609 ++++++ tests/test_evals_debias_verifiers.py | 1031 ++++++++++ tests/test_evals_drivers.py | 998 +++++++++ tests/test_evals_hardening.py | 974 +++++++++ tests/test_evals_proxy.py | 1795 +++++++++++++++++ tests/test_evals_report_ops.py | 525 +++++ tests/test_evals_surface.py | 165 ++ tests/test_evals_verifiers.py | 731 +++++++ 23 files changed, 16463 insertions(+) create mode 100644 evals/DESIGN.md create mode 100644 evals/README.md create mode 100644 evals/__init__.py create mode 100644 evals/cleanup.py create mode 100644 evals/drivers.py create mode 100755 evals/env.sh create mode 100644 evals/listing.py create mode 100644 evals/mock_flags.py create mode 100644 evals/proxy.py create mode 100644 evals/report.py create mode 100644 evals/run.py create mode 100644 evals/seed.py create mode 100644 evals/tasks.py create mode 100644 tests/test_evals_catalog.py create mode 100644 tests/test_evals_debias_verifiers.py create mode 100644 tests/test_evals_drivers.py create mode 100644 tests/test_evals_hardening.py create mode 100644 tests/test_evals_proxy.py create mode 100644 tests/test_evals_report_ops.py create mode 100644 tests/test_evals_surface.py create mode 100644 tests/test_evals_verifiers.py diff --git a/.gitignore b/.gitignore index 88f9cb18..c771e92d 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,12 @@ htmlcov/ .tox/ .hypothesis/ +# Eval harness output + local env bootstrap state +evals/results/ +evals/.env-pids +evals/.api_runserver.log +evals/.mock_flags.log + # Mypy .mypy_cache/ .dmypy.json diff --git a/evals/DESIGN.md b/evals/DESIGN.md new file mode 100644 index 00000000..9ac30359 --- /dev/null +++ b/evals/DESIGN.md @@ -0,0 +1,358 @@ +# Plane MCP Tool-Surface Eval Harness + +Measures how well an LLM agent completes real Plane tasks through this MCP server's tool +surface. Produces decision-grade numbers for the tool-consolidation discussion: success rate, +tool calls to done, wrong-tool picks, and per-call response token cost. + +This document is the full spec. Phase 1 (walking skeleton) implements a subset — see +"Phase 1 scope" at the bottom. + +## Why + +The live surface is 177 tools. A consolidation proposal (139→47) exists, but its cost +claims were estimate-based and wrong by 3.4× when measured. Before reshaping anything we +need empirical answers to: + +1. **Mispick rate** — how often does an agent choose the wrong tool among overlapping ones + (7 list-variants for work items, links vs relations, etc.)? +2. **Calls-to-done vs optimal** — how much does the name→UUID resolution dance and + sub-object fan-out (item + comments + links as separate calls) cost? +3. **Response bloat** — how many tokens does each tool result actually inject into context? + +The harness must support A/B comparison: same tasks against different tool surfaces +(`full` today; later `core` tag-filter and `v2` transform-layer variants). + +## Architecture + +**Driver:** the Anthropic Python SDK's beta tool runner with its MCP conversion helpers — +NOT the Claude Agent SDK, NOT a hand-rolled agent loop. This measures the MCP surface in +isolation (no coding-harness system prompt or built-in tools polluting the numbers). + +The exact pattern (this is documented SDK API — do not improvise alternatives): + +```python +from anthropic import AsyncAnthropic +from anthropic.lib.tools.mcp import async_mcp_tool +from mcp import ClientSession +from mcp.client.stdio import stdio_client, StdioServerParameters + +client = AsyncAnthropic() # resolves ANTHROPIC_API_KEY / ant-auth profile from env + +server_params = StdioServerParameters( + command=sys.executable, + args=["-m", "plane_mcp", "stdio"], + env={ + "PLANE_API_KEY": os.environ["EVAL_PLANE_API_KEY"], + "PLANE_WORKSPACE_SLUG": os.environ["EVAL_PLANE_WORKSPACE_SLUG"], + "PLANE_BASE_URL": os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so"), + }, +) + +async with stdio_client(server_params) as (read, write): + async with ClientSession(read, write) as mcp_client: + await mcp_client.initialize() + tools_result = await mcp_client.list_tools() + runner = client.beta.messages.tool_runner( # sync call — returns the runner + model=MODEL_ID, + max_tokens=8192, + max_iterations=15, # turn cap per task + system=SYSTEM_PREAMBLE, + messages=[{"role": "user", "content": task["prompt"]}], + tools=[async_mcp_tool(t, mcp_client) for t in tools_result.tools], + ) + async for message in runner: + # capture tool_use blocks from message.content + for block in message.content: + if block.type == "tool_use": + record_call(block.name, block.input) + # capture tool results (cached — tools still run exactly once) + tool_response = runner.generate_tool_call_response() + if tool_response is not None: + record_results(tool_response) # user message w/ tool_result blocks + final = message +``` + +Notes: +- One fresh stdio server subprocess per task run (cheap, isolates state). +- Record `message.usage` from **every** yielded message (input_tokens, output_tokens, + cache_read_input_tokens, cache_creation_input_tokens) — this is the exact context cost, + returned free; the per-result counts below are a size proxy, not the cost figure. +- Record the final message's `stop_reason`, and whether the loop ended by exhausting + `max_iterations` — a capped/truncated run must be distinguishable from a genuine failure. + Detect the cap from `stop_reason` (the runner can legitimately finish with `end_turn` on + exactly its last permitted iteration — an unconditional iteration-count check misreports + that as capped). +- **Never call `generate_tool_call_response()` on a refusal-terminated message** + (`stop_reason == "refusal"`): the SDK deliberately skips executing those tool_use blocks + (side effects the model never confirmed), and calling it from the loop body bypasses that + guard and fires real writes at the eval workspace. +- `wall_time_s` measures the agent loop only: start the clock after `list_tools()` returns, + stop it when the loop exits — MCP subprocess spawn/teardown and post-loop token counting + are excluded. +- Build the stdio server env **from scratch** (`PATH`, `HOME`, plus exactly the three + `PLANE_*` vars) — never inherit `os.environ`. `plane_mcp/client.py` prefers + `PLANE_INTERNAL_BASE_URL` over `PLANE_BASE_URL`, so an inherited value silently points + the agent at a different Plane instance than seed/verify. +- A harness/API failure (SDK exception, MCP crash) is recorded as `error: ""` on the + row — it is neither a task failure nor a skip, and the row's zeroed metrics must not + enter any statistic. +- `SYSTEM_PREAMBLE` names the eval workspace slug and project name, states "complete the + task using the available tools, then stop", and nothing else. Keep it under 100 words — + it is part of the measured context. +- Omit `thinking` and sampling params entirely (adaptive thinking is the default on + claude-sonnet-5; `temperature`/`top_p`/`top_k` are rejected). +- Final assistant text = the last yielded message's text blocks (used by read-task verifiers). + +**Token counting of tool results:** use the API's count_tokens endpoint, never tiktoken +(wrong tokenizer for Claude, ~15-20% off): + +```python +n = ( + await client.messages.count_tokens( + model=MODEL_ID, + messages=[{"role": "user", "content": result_text}], + ) +).input_tokens +``` + +Rules: +- Run these counts **after** the agent loop finishes, not inline — they must not pollute + `wall_time_s`. Buffer the raw result strings during the run, count at the end. +- A tool_result's content may be a list of blocks. Concatenate the text of `text` blocks; + for non-text blocks (e.g. image) record `result_kind: "image"` with `result_tokens: null` + and `result_chars` of the raw payload. `is_error` results are counted like text. +- Also record raw `len(chars)` alongside every count. + +**Models** (CLI aliases → IDs; these are deliberate, do not substitute): + +| alias | model id | role | +|---|---|---| +| `sonnet` | `claude-sonnet-5` | default / representative agent | +| `haiku` | `claude-haiku-4-5` | canary — weaker models amplify tool-surface defects | + +## Environment + +| var | purpose | +|---|---| +| `ANTHROPIC_API_KEY` | driver LLM auth (or an `ant auth` profile) | +| `EVAL_PLANE_API_KEY` | Plane API key for the **dedicated eval workspace** | +| `EVAL_PLANE_WORKSPACE_SLUG` | eval workspace slug — never a production workspace | +| `EVAL_PLANE_BASE_URL` | optional, defaults to `https://api.plane.so` | + +Seeding and verification talk to Plane directly via `plane-sdk` (already a dependency). +Construct the client the same way `plane_mcp/client.py` does for stdio mode, but from the +`EVAL_*` vars. + +## Files + +``` +evals/ + __init__.py + DESIGN.md # this file + tasks.py # task definitions (plain dicts) + verifier functions + seed.py # per-run fixture create/teardown via plane-sdk + run.py # CLI driver (python -m evals.run) + report.py # summary table + A/B delta (python -m evals.report) + results/ # *.jsonl output — gitignored +``` + +Dependencies: add to `pyproject.toml`: + +```toml +[project.optional-dependencies] +evals = ["anthropic[mcp]>="] +``` + +Pin a `>=` floor at whatever the current anthropic release is when you implement, and +verify `from anthropic.lib.tools.mcp import async_mcp_tool` actually imports in the venv. +Do NOT change the existing `mcp==1.26.0` pin — `anthropic[mcp]` must coexist with it. +No other new dependencies. Stdlib only otherwise (argparse, json, asyncio, uuid, time). + +## Task schema (`tasks.py`) + +Plain dicts, no classes: + +```python +{ + "id": "W1", + "tags": {"write", "tier1"}, + "prompt": "Create a work item in project {project}: title 'Login page 500s on empty " + "password', priority urgent, assign it to me, and add the 'auth' label.", + "optimal_calls": 4, + "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, + "alternate_tools": { + "search_work_items", + "list_states", + "retrieve_project", + "get_workspace_members", + "manage_work_item_assignee", + "manage_work_item_label", + "update_work_item", + }, + "needs": {"labels"}, # fixture groups seed.py must create + "verify": verify_w1, # async (plane, ctx, run) -> (bool, note) +} +``` + +- `{project}` in prompts is formatted with the seeded project name at runtime. +- `optimal_tools` and `alternate_tools` are **disjoint** sets. Every call is classified as + one of `optimal` / `alternate` / `out_of_set`, plus an independent `is_error` flag. + **Mispick = alternate + out_of_set** — this is the eval's headline metric, so authoring + matters: a tool that works but is the *wrong pick among overlapping variants* (a list + variant where search is optimal, a link where a relation is asked for) belongs in + `alternate_tools` or nowhere, never in `optimal_tools`. The full ordered call list is + kept in the JSONL so classifications can be re-derived offline if sets are revised. +- **Action-dispatch surfaces need a finer mispick unit** (added 2026-08-11, for the P2 + A/B that compares PR #195's 29-tool `action`-multiplexer variant): on such a surface + the model almost always picks the "right" *tool* and fails inside it — wrong `action`, + or params invalid for the chosen action. Tool-name classification alone would + under-count exactly that failure mode and bias the A/B toward consolidation. Rule: when + a surface under test multiplexes verbs through a parameter, the classification unit is + `(tool, action)` — task authors list optimal/alternate *(tool, action)* pairs — and a + schema-valid call whose params are invalid for its declared action counts as + `out_of_set`, not merely `is_error`. Flat surfaces are unaffected (their action unit is + the tool name). The stored ordered call list already carries arguments, so this scoring + can also be re-derived retroactively. +- `verify` receives `run = {"final_text": str, "calls": [...]}` alongside the plane client + and seed ctx. Write tasks assert end state through the Plane API; read tasks match the + final text against seeded facts using **word-boundary regexes on exact seeded values** + (each verifier states its matching rule in a comment — naive substring matching is a + known false-positive source, e.g. bare "4" inside "24"). Resolve expected values via + API at verify time — never hardcode sequence numbers or UUIDs. + +## Fixtures (`seed.py`) + +Per run: create project named `EVAL {run8}` (`run8` = first 8 hex chars of `uuid4().hex`; +identifier `EV` + 4 of those hex chars uppercased, ≤12 chars) in the eval workspace. +Unique-per-run naming makes runs parallel-safe and crash-visible. Provide +`seed(plane, run_id, needs) -> ctx` and `teardown(plane, ctx)`; `ctx` carries project_id, +project name, and IDs of everything seeded. + +`seed()` must guarantee teardown information even on partial failure: it mutates a +caller-provided ctx in place (or raises with the partial ctx attached), so a failure +after project creation never leaks an untracked project. Feature probes must read the +keys the API actually returns (workspace toggle: `is_work_item_types_enabled`) and a +seed failure must fail loudly — never masquerade as a plan-gate skip. + +The R1 target item is seeded into a **non-default state** (e.g. a `started`-group state) +so a guessed default state name cannot pass verification. + +Teardown deletes the project **plus every workspace-scoped object seeded** — customers +(and any other object that survives project deletion) are tracked in `ctx` and deleted by +ID explicitly; project deletion alone is not sufficient cleanup. + +Seed only the fixture groups the selected tasks declare in `needs`: + +| group | contents | +|---|---| +| `items` | ~12 work items with fixed titles/priorities incl. "Payment webhook drops retries" (urgent); exactly 4 urgent open items total | +| `labels` | labels `auth`, `triage`, `perf` | +| `cycles` | "Sprint 12" (past-dated), "Sprint 13" (current) | +| `module` | "Checkout revamp" with 3 completed items | +| `bug_type` | work item type "Bug" — **plan-gated feature**: if the API rejects creation, seed() records `bug_type: None` and dependent tasks are SKIPPED (recorded in JSONL with `"skipped": reason`), not failed | +| `intake` | 2 intake items (one billing request, one obvious spam) | +| `customer` | customer "Acme Corp" + request "SSO support" | +| `release` | release "1.2.0" with 2 changelog entries | + +## Runner CLI (`run.py`) + +``` +python -m evals.run --tasks R1,W1,S1 --model sonnet --reps 1 \ + --surface full --out evals/results/.jsonl +python -m evals.run --list # print task table, no network +python -m evals.run --dry-run --tasks R1 # print resolved prompt + seed plan, no network +``` + +- `--surface` is recorded in the JSONL and (for now) only `full` is implemented; it is the + future hook for tag-filtered/transformed variants. Unknown values error. +- `--reps N` repeats each task N times (fresh seed + fresh server per rep). +- Per task-rep flow: seed → run agent → verify → append JSONL row → teardown (teardown in + a `finally`; on crash, print the orphaned project name). + +One JSONL row per task-rep: + +```json +{"run_id": "...", "ts": "...", "git_sha": "...", "surface": "full", + "model": "claude-sonnet-5", "task_id": "W1", "rep": 0, + "success": true, "verify_note": "...", "skipped": null, "error": null, + "stop_reason": "end_turn", "hit_max_iterations": false, + "calls": [{"tool": "list_projects", "class": "optimal", "args_chars": 42, + "result_tokens": 830, "result_chars": 3120, "result_kind": "text", + "is_error": false}], + "num_calls": 4, "errored_calls": 0, "alternate_calls": 0, "out_of_set_calls": 0, + "total_result_tokens": 2210, + "usage_per_iteration": [{"in": 38210, "out": 412, "cache_read": 36100, "cache_write": 0}], + "cum_input_tokens": 152840, "wall_time_s": 31.4} +``` + +## Report (`report.py`) + +``` +python -m evals.report evals/results/A.jsonl [evals/results/B.jsonl] +``` + +Single file, per task: `n`, success as `k/n` with a 95% Wilson interval, median calls +(with IQR) vs optimal, mispick rate (alternate + out_of_set over total calls), errored +calls, capped runs (`hit_max_iterations` or `stop_reason == "max_tokens"`) and harness-error +rows (`error != null`) each reported as their own column — never silently folded into +failures, and error rows excluded from success/medians entirely — median & p95 result_tokens per +call, and median `cum_input_tokens`. + +Two files: same table with per-task deltas (B − A). **Refuse the delta mode (exit with a +message) when either file has n < 5 for any shared task** — comparative claims below that +floor are noise. Plain text, stdlib only. + +Optimal-path caveats discovered during review (bake into the sets and a comment): +- **R1**: `search_work_items` returns no state field (`WorkItemSearchItem` has only + name/id/sequence_id/identifiers). The true 1-call path is `list_work_items` (WorkItem + carries `state: str | StateLite`; `expand=state` yields the name). search is alternate. +- **S1**: `create_work_item_property` accepts inline `options`, so the optimal path is + 3 calls (`list_projects` → `resolve_work_item_type` → `create_work_item_property`) — + separate option-creation / list_work_item_types calls are alternates, not optimal. + +## Full task list (target: 20 tasks) + +Defined in the consolidation analysis; implement incrementally. IDs are stable. + +| id | prompt (abbrev) | probes | optimal | +|---|---|---|---| +| R1 | state of item titled 'Payment webhook drops retries' | list vs search (search has no state) | 1 (`list_work_items`) | +| R2 | how many urgent open items | count/list/search pick, UUID dance | 1–2 | +| R3 | items assigned to me due this week | assignee resolution | 2 | +| R4 | what's in the active cycle, anything overdue | PQL activeCycle() discovery | 1–2 | +| R5 | summarize discussion on a known item | sub-object fan-out | 2 | +| R6 | which project has more open bugs (needs 2nd project) | cross-project composition | 2–3 | +| W1 | file a bug w/ priority+assignee+label | lookup overhead before create | 3–4 | +| W2 | move item to Done | state name→UUID | 2–3 | +| W3 | comment on an item | happy path baseline | 2 | +| W4 | rename label triage→needs-triage | update_label pick | 2 | +| W5 | archive all completed items in module | no-bulk N-call burn | 2+N | +| W6 | move unfinished items Sprint 12→13, close Sprint 12 | transfer+complete workflow | 3–4 | +| W7 | mark A blocking B + add reference URL | relations-vs-links confusion | 3 | +| W8 | log 2h on an item for yesterday | worklog | 2 | +| S1 | add Severity dropdown (Critical/Major/Minor) to Bug type | property + inline options | 3 | +| S2 | add Fibonacci estimate scale, set item to 5 pts | estimates chain | 4–5 | +| S3 | create type Incident w/ required text property | type+property+attach | 4–5 | +| S4 | triage intake: accept billing, reject spam | workflow vs endpoint tools | 3–4 | +| C1 | create customer Acme, link request to item | customers domain | 3–4 | +| C2 | what shipped in release 1.2.0 | releases/changelog | 1–2 | + +## Phase 1 scope (walking skeleton) + +Implement end to end, nothing more: + +1. `tasks.py` with **R1, W1, S1 only** (verifiers included). +2. `seed.py` covering fixture groups `items`, `labels`, `bug_type`. +3. `run.py` with `--list`, `--dry-run`, and the full live path (seed→run→verify→teardown). +4. `report.py` single-file mode (A/B delta mode can be a stub that errors clearly). +5. `pyproject.toml` evals extra + `evals/results/` gitignored. + +Constraints: +- Python 3.10+, ruff clean (`ruff format evals/ && ruff check evals/` — line length 120, + rules E,F,I,UP,B per pyproject). +- Match the codebase's existing style; no classes where dicts do, no framework. +- **No live credentials exist in this checkout** — done means: `--list` and `--dry-run` + work without network, imports resolve in a fresh venv after + `uv pip install -e ".[dev,evals]"`, ruff passes. The live path must be complete and + plausible but cannot be executed yet. +- Do NOT commit. Do NOT touch `.ccwrc`, `.env*`, or anything under `plane_mcp/`. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 00000000..0dc4494c --- /dev/null +++ b/evals/README.md @@ -0,0 +1,176 @@ +# Eval harness — runbook + +Measures how well an LLM agent completes real Plane tasks through an MCP tool surface. +Every task runs against a **live** Plane API with fixtures seeded and torn down per run, +and is graded by a verifier that reads the API back — not by inspecting the agent's prose. + +The harness is agent-agnostic and surface-agnostic: any stdio MCP server can be measured +(`--server-cmd`), driven by any of five agent backends. `DESIGN.md` explains why it is +built this way; this file is how to run it. + +What you get per task: pass/fail, tool calls to done, which tools were picked (and whether +they were the optimal ones), errors, and the agent's final text. + +## Prerequisites + +1. **A local Plane API.** `evals/env.sh` starts plane-ee on `:8000` plus a mock + feature-flag server on `:9911` that turns every flag on. + + ```bash + export PLANE_EE_API_DIR=/path/to/plane-ee/apps/api + export PLANE_EE_VENV=/path/to/plane-ee-venv + evals/env.sh up # down | status + ``` + +2. **A workspace and an API key** on that instance, with a Business/Enterprise license so + the gated fixtures (customers, releases) can be seeded. + + ```bash + export EVAL_PLANE_BASE_URL=http://localhost:8000 + export EVAL_PLANE_WORKSPACE_SLUG= + export EVAL_PLANE_API_KEY=plane_api_... + unset REDIS_HOST REDIS_PORT # else the SDK client picks up a stale cache config + ``` + +3. **An agent CLI** for the driver you pick (below), already authenticated. + +## Running + +```bash +# Everything, one surface +.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ + --surface full --out results/legacy.jsonl + +# A few tasks while iterating +.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ + --surface full --tasks W5,W8 --out results/spot.jsonl + +# Someone else's server (a PR branch, another repo) — "external mode" +.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ + --surface their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ + --server-env PLANE_MCP_TOOLS_VERSION=v2 --out results/their-pr.jsonl +``` + +Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed +`(task, rep)` pairs, retry only infra failures), `--list` / `--dry-run` (no network). + +**External mode** (`--server-cmd`) runs every task with no surface-based skips, and turns +off mispick classification — foreign tool names have no optimal/alternate sets to score +against, so call *counts* stay comparable but "mispicks" reads `n/a`. + +**On surfaces.** `--surface` without `--server-cmd` runs this repo's own server, and passes +the surface through as `PLANE_MCP_SURFACE`. This branch's server serves one surface, so +only `full` is real here: `v2` / `v2-schema` set an env var nothing reads, and you would +get legacy results labelled as something else. The task catalog keeps its per-surface +overlays (`surface_tools`) so those surfaces score correctly if the server ever grows them. +**Measure any other surface through `--server-cmd`** — that is how the PR surfaces were +compared, and it is honest about what it ran because it launches the server you name. + +### Drivers + +| Driver | Backend | Notes | +|---|---|---| +| `codex-cli` | OpenAI Codex CLI | Pass a real model id (`gpt-5.6-sol`); the short-alias table is incomplete | +| `claude-cli` | Claude Code CLI | `--model sonnet` / `haiku` | +| `antigravity-cli` | Antigravity CLI (`agy`) | Runs under a synthetic HOME so its MCP config is ours, not yours | +| `opencode-cli` | opencode | Temp project config per run | +| `sdk` | Anthropic SDK tool runner | No coding-harness prompt in the way; needs `ANTHROPIC_API_KEY` | + +Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool +calls are counted from the wire rather than from whatever the agent claims it did. + +### Reading results + +```bash +.venv/bin/python -m evals.report results/legacy.jsonl # one surface +.venv/bin/python -m evals.report --table results/*.jsonl # side by side +.venv/bin/python -m evals.report --table --markdown results/*.jsonl # for a PR +``` + +Rows are deduped latest-wins per `(task_id, rep, surface)`, so a re-run of a single task +supersedes its earlier row in the same file. Skipped tasks are excluded from success +denominators — a surface that cannot do something is not punished as a failure, it is +reported as a skip. + +## Running surfaces in parallel + +Tasks that touch **workspace-scoped** fixtures (release tags, customer properties) collide +if two runs share a workspace. Give each concurrent run its own workspace: + +```bash +EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --surface full --out results/legacy.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --surface their-pr --out results/their-pr.jsonl & +wait +``` + +Keep the workspaces **empty apart from eval fixtures**. Unrelated projects and work items +in one workspace but not another skew every workspace-wide task in that column. + +## Adding a task + +Tasks live in `evals/tasks.py`. A task is a dict: + +```python +{ + "id": "W11", + "tags": {"write", "tier1"}, + "prompt": f"In project {{project}}, ...", # {project} is bound at run time + "optimal_calls": 3, + "optimal_tools": {"list_cycles", "complete_cycle"}, # scored as optimal picks + "alternate_tools": {"list_projects"}, # acceptable, not optimal + "surface_tools": {"v2": {"optimal_tools": {"close_cycle"}}}, # per-surface overlay + "needs": {"items", "cycles"}, # fixtures to seed + "verify": verify_w11, +} +``` + +`needs` tokens: `items`, `labels`, `bug_type`, `cycles`, `cycles_open_past`, `module`, +`intake`, `customer`, `release`, `activity_feed`, `second_project`, +`leave_cycles_worklogs_off`. Each task gets its own freshly seeded project, so fixture +variants (e.g. `cycles_open_past`) don't leak between tasks. + +A `surface_tools` overlay may set `"expected_skip": True` to declare that a surface +genuinely cannot do the task. That is reported as a capability gap, not a failure. + +### Writing a verifier + +Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)` and must read +state back through the API. Two rules, both learned the hard way: + +**Never parse natural language.** Constrain the output in the prompt instead — end the +prompt with `Answer with a line 'count: N'` and match that line exactly. Regex over prose +produces both false passes ("10 attachments" satisfying a truth of 0) and false failures +("three" for 3), and no amount of tuning fixes it. + +**Check the shape the API actually returns.** Dates come back as timestamps +(`2026-08-12T00:00:00Z`), so comparing one to a bare `2026-08-12` silently never matches — +a verifier that can only fail is worse than no verifier. Have the test stub return the real +shape. + +Then prove the verifier can fail: + +```bash +.venv/bin/python -m evals.run --canary --surface full +``` + +The canary seeds every task, calls each verifier with an **empty** agent result, and exits +non-zero if any verifier passes a do-nothing agent. Run it after touching tasks, fixtures, +or verifiers. + +**Make the task achievable before blaming a surface.** A fixture that forbids what the +prompt asks turns the task into a coin flip on tool choice and will implicate whichever +server happens to pick differently. W6 was pre-closing a cycle the agent was asked to +close; it took two full runs to notice, because a side effect of an unrelated call was +tripping the verifier. + +## Local gotchas + +- **Comment activity never appears** without a running activity worker, so the tasks that + read the activity feed self-skip (`env:no-activity-worker`) rather than fail. +- **A feature-flag cache poisoned by the wrong disco.** Anything that talks to the DB while + sourcing `plane-ee/apps/api/.env` uses the *remote* flag server (all flags off) and + caches that answer for a workspace the API server would otherwise serve from the local + mock. Symptom: gated endpoints 402 and the canary reports every verifier broken. Fix: + clear `ff::*` and rotate `ff_ver:` (`plane.payment.flags.cache`). +- **Offline tests** cover the harness itself and need no Plane instance: + `env -u REDIS_HOST -u REDIS_PORT .venv/bin/python -m pytest -q --ignore=tests/test_integration.py` diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 00000000..fb5f7791 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""Plane MCP tool-surface eval harness.""" diff --git a/evals/cleanup.py b/evals/cleanup.py new file mode 100644 index 00000000..8f6f4246 --- /dev/null +++ b/evals/cleanup.py @@ -0,0 +1,109 @@ +"""Delete leftover projects whose names start with a space-delimited prefix (default ``"EVAL "``). + +Usage: + python -m evals.cleanup # dry-run: list only + python -m evals.cleanup --prefix "EVAL " # custom name prefix (note trailing space) + python -m evals.cleanup --yes # actually delete + +Uses EVAL_PLANE_API_KEY / EVAL_PLANE_WORKSPACE_SLUG via make_plane_client. +Default is dry-run; ``--yes`` is required to call delete. +""" + +from __future__ import annotations + +import argparse +import sys +from typing import Any + +from plane.models.query_params import PaginatedQueryParams + + +def list_projects_with_prefix(plane: Any, workspace_slug: str, prefix: str) -> list[Any]: + """Return projects whose name starts with ``prefix`` (paginated list). + + Matches the SDK contract used elsewhere in the repo: pass + ``params=PaginatedQueryParams(...)`` and stop when ``not page.next_page_results``. + Do not fall back on ``next_cursor`` alone — the SDK always populates it. + """ + matches: list[Any] = [] + cursor = None + while True: + params = PaginatedQueryParams(per_page=100, cursor=cursor) + page = plane.projects.list(workspace_slug=workspace_slug, params=params) + results = page.results if hasattr(page, "results") else page + for proj in results or []: + # Prefix may include a trailing space (default "EVAL ") so "EVALUATION" is excluded. + name = getattr(proj, "name", None) or "" + if name.startswith(prefix): + matches.append(proj) + if not getattr(page, "next_page_results", False): + break + cursor = page.next_cursor + return matches + + +def delete_projects( + plane: Any, + workspace_slug: str, + projects: list[Any], + *, + yes: bool, +) -> tuple[int, int]: + """Delete projects when yes=True. Returns (deleted, failed). Dry-run: (0, 0).""" + if not yes: + return 0, 0 + deleted = failed = 0 + for proj in projects: + pid = getattr(proj, "id", None) + name = getattr(proj, "name", pid) + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=pid) + deleted += 1 + print(f" deleted {name!r} ({pid})") + except Exception as exc: + failed += 1 + print(f" FAILED {name!r} ({pid}): {exc}", file=sys.stderr) + return deleted, failed + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description="Clean up leftover EVAL projects (dry-run by default)") + p.add_argument("--prefix", type=str, default="EVAL ", help='Project name prefix (default: "EVAL ")') + p.add_argument( + "--yes", + action="store_true", + help="Actually delete matched projects (default is dry-run list only)", + ) + args = p.parse_args(argv) + + from evals.seed import make_plane_client + + try: + plane, workspace_slug = make_plane_client() + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + projects = list_projects_with_prefix(plane, workspace_slug, args.prefix) + print(f"workspace={workspace_slug} prefix={args.prefix!r} matches={len(projects)}") + for proj in projects: + pid = getattr(proj, "id", "?") + name = getattr(proj, "name", "?") + ident = getattr(proj, "identifier", "") + print(f" {name!r} id={pid} identifier={ident}") + + if not projects: + print("nothing to delete") + return 0 + + if not args.yes: + print(f"dry-run: would delete {len(projects)} project(s); re-run with --yes to delete") + return 0 + + deleted, failed = delete_projects(plane, workspace_slug, projects, yes=True) + print(f"summary: deleted={deleted} failed={failed} matched={len(projects)}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/drivers.py b/evals/drivers.py new file mode 100644 index 00000000..0de55bc6 --- /dev/null +++ b/evals/drivers.py @@ -0,0 +1,1947 @@ +"""Agent-driver abstraction for the Plane MCP eval harness. + +Drivers run one task against a tool surface and return a normalized +``AgentRun``. The default ``sdk`` driver preserves the historical Anthropic +SDK + in-process MCP client path. CLI drivers (``claude-cli``, ``codex-cli``) +spawn locally installed agent CLIs on the user's subscription — no Anthropic +API key required for those paths. + +Real CLI surfaces (probed on this machine, 2026-08-12): + +Claude Code (``claude`` v2.1.228): + - ``-p`` / ``--print`` headless + - ``--mcp-config `` (repeatable; ``--strict-mcp-config``) + - ``--output-format json|text|stream-json`` (print mode) + - ``--max-turns `` (print mode; *hidden* from ``--help`` but present) + - ``--model `` + - ``--permission-mode`` choices: acceptEdits, auto, bypassPermissions, + manual, dontAsk, plan + - ``--dangerously-skip-permissions``, ``--allowedTools`` / ``--allowed-tools`` + - Transcript: ``~/.claude/projects//.jsonl`` + with ``assistant`` rows whose ``message.content`` holds ``tool_use`` blocks. + - MCP tools surface as ``mcp____`` — strip for classification. + +Codex (``codex exec``): + - ``codex exec --json`` JSONL events on stdout + - ``-c key=value`` / ``--config`` for config.toml overrides (incl. mcp_servers) + - ``-m`` / ``--model`` + - Session rollouts: ``~/.codex/sessions/**/rollout-*.jsonl`` with + ``response_item`` / ``function_call`` payloads (name + arguments JSON string) + - Marked **experimental**; live runs are opt-in (metered quota). +""" + +from __future__ import annotations + +import json +import os +import re +import signal +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# mcp__plane__list_work_items → list_work_items +# mcp__plane-mcp-server__foo → foo +_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") +# Alternate: mcp__server__tool with multi-segment server names +_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") + + +def proxy_wrap_server_command( + real_command: list[str], + *, + sidecar_path: Path, + python_bin: str | None = None, +) -> list[str]: + """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" + py = python_bin or sys.executable + return [py, "-m", "evals.proxy", "--log", str(sidecar_path), "--", *real_command] + + +def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: + """Inject the repo root into PYTHONPATH so ``python -m evals.proxy`` works from any cwd. + + ``evals`` is not an installed package (pyproject excludes it); the MCP child + is often launched from a foreign temp dir (OpenCode project dir, etc.). + """ + root = str(REPO_ROOT) + out = dict(env) + existing = out.get("PYTHONPATH", "") + parts = [p for p in existing.split(os.pathsep) if p] + if root not in parts: + out["PYTHONPATH"] = root + (os.pathsep + existing if existing else "") + return out + + +def load_proxy_sidecar( + path: Path, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Load sidecar call rows (sorted by seq) plus a status dict. + + Status keys: + - missing / empty / complete / incomplete + - torn_line: final line failed to parse + - meta: proxy_meta row if present + - pending_left: from meta when present + """ + status: dict[str, Any] = { + "state": "missing", + "torn_line": False, + "meta": None, + "pending_left": None, + } + if not path.is_file(): + return [], status + try: + raw = path.read_bytes() + except OSError: + return [], status + if not raw: + status["state"] = "empty" + return [], status + + # Decode with replacement so invalid UTF-8 does not crash the loader. + text = raw.decode("utf-8", errors="replace") + lines = text.splitlines() + calls: list[dict[str, Any]] = [] + meta: dict[str, Any] | None = None + torn = False + for i, line in enumerate(lines): + s = line.strip() + if not s: + continue + try: + row = json.loads(s) + except json.JSONDecodeError: + # Tolerate a torn final line (crash mid-write); stop there. + if i == len(lines) - 1: + torn = True + break + continue + if not isinstance(row, dict): + continue + if row.get("row_type") == "proxy_meta": + meta = row + continue + tool = row.get("tool") + if not tool: + continue + calls.append( + { + "tool": str(tool), + "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), + "origin": "plane", + "is_error": bool(row.get("is_error")), + "result_chars": int(row.get("result_chars") or 0), + "duration_ms": row.get("duration_ms"), + "seq": row.get("seq"), + } + ) + + # Score order must match request seq, not response-append order. + calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) + + status["torn_line"] = torn + status["meta"] = meta + if meta is not None: + status["pending_left"] = meta.get("pending_left") + status["pumps_alive"] = bool(meta.get("pumps_alive")) + incomplete = bool( + torn + or meta is None + or (meta is not None and int(meta.get("pending_left") or 0) > 0) + or (meta is not None and bool(meta.get("pumps_alive"))) + ) + if not calls and not meta and not torn: + status["state"] = "empty" + elif incomplete: + status["state"] = "incomplete" + else: + status["state"] = "complete" + return calls, status + + +def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: + """Convenience: call rows only (sorted by seq).""" + calls, _status = load_proxy_sidecar(path) + return calls + + +def apply_proxy_sidecar( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: + """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. + + Incomplete sidecar (torn line, missing meta, pending_left>0) yields to the + CLI trace when the CLI has *more* plane calls. Returns + ``(plane_calls, client_calls, call_source)``. + """ + proxy_calls, status = load_proxy_sidecar(sidecar_path) + state = status.get("state") + if state in ("missing", "empty"): + notes.append("proxy_sidecar_empty") + return calls, client_calls, "json" + if state == "incomplete": + notes.append( + "proxy_sidecar_incomplete" + + (":torn" if status.get("torn_line") else "") + + (":no_meta" if status.get("meta") is None else "") + + (f":pending_left={status.get('pending_left')}" if status.get("pending_left") else "") + + (":pumps_alive" if status.get("pumps_alive") else "") + ) + if len(calls) > len(proxy_calls): + notes.append("proxy_sidecar_deferred_to_cli_trace") + return calls, client_calls, "json" + if proxy_calls: + notes.append(f"calls_from_proxy:{sidecar_path}") + return proxy_calls, client_calls, "proxy" + return calls, client_calls, "json" + # complete + notes.append(f"calls_from_proxy:{sidecar_path}") + return proxy_calls, client_calls, "proxy" + + +def wait_for_proxy_meta( + sidecar_path: Path, + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> bool: + """Poll until the sidecar gains a ``proxy_meta`` row (or the wait expires). + + After a CLI timeout the driver kills the CLI; the proxy is a *separate* + process that only then sees stdin EOF and needs up to + ``SHUTDOWN_DEADLINE_S`` to flush call rows + meta. Call this **before** + harvesting so the temp dir is not deleted mid-finalization. + + Returns True if meta was observed. + """ + # Local import keeps drivers import-light for non-proxy unit tests. + from evals.proxy import SHUTDOWN_DEADLINE_S + + if max_wait_s is None: + max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 + deadline = time.monotonic() + max_wait_s + while True: + _, status = load_proxy_sidecar(sidecar_path) + if status.get("meta") is not None: + return True + rem = deadline - time.monotonic() + if rem <= 0: + break + time.sleep(min(poll_s, rem)) + _, status = load_proxy_sidecar(sidecar_path) + return status.get("meta") is not None + + +def harvest_proxy_after_cli_timeout( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], + *, + max_wait_s: float | None = None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: + """Wait for proxy finalization after CLI kill, then harvest the sidecar. + + If meta never appears within the wait window, harvest anyway (incomplete + note from ``apply_proxy_sidecar``). ``max_wait_s`` defaults to + ``SHUTDOWN_DEADLINE_S + 2`` (see ``wait_for_proxy_meta``). + """ + found = wait_for_proxy_meta(sidecar_path, max_wait_s=max_wait_s) + if not found: + notes.append("proxy_meta_wait_timeout") + return apply_proxy_sidecar(calls, client_calls, sidecar_path, notes) + + +# Bounded drain after process-group kill so communicate() never hangs forever +# when a grandchild still holds the pipe open. +_CLI_TIMEOUT_DRAIN_S = 2.0 + + +def _kill_process_group(proc: subprocess.Popen[Any]) -> bool: + """SIGKILL the process group whose leader is ``proc``. + + With ``start_new_session=True``, ``pgid == proc.pid`` even after the leader + has been reaped — call ``killpg(proc.pid, …)`` directly (never fall back to + killing only the leader, which leaves grandchildren alive). + + Returns True if ``killpg`` delivered the signal; False if the group is + already fully gone (``ProcessLookupError`` = success for cleanup, but the + kill itself did not run). + """ + if proc.pid is None: + return False + try: + # Do NOT use getpgid: if the leader is already reaped, getpgid fails and + # a proc.kill() fallback would recreate the original orphan bug. + os.killpg(proc.pid, signal.SIGKILL) + return True + except ProcessLookupError: + # No process left in the group — fully gone. + return False + + +def _decode_pipe(data: str | bytes | None, *, text: bool) -> str | bytes | None: + if data is None or not text or isinstance(data, str): + return data + return data.decode("utf-8", errors="replace") + + +def _close_pipes_and_reap(proc: subprocess.Popen[Any], *, drain_s: float = _CLI_TIMEOUT_DRAIN_S) -> tuple[Any, Any]: + """Bounded drain / close after a group kill. Never hangs unbounded.""" + try: + return proc.communicate(timeout=drain_s) + except (subprocess.TimeoutExpired, ValueError, OSError): + for stream in (proc.stdout, proc.stderr): + if stream is not None: + try: + stream.close() + except Exception: + pass + try: + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + pass + return None, None + + +def run_cli_subprocess( + cmd: list[str], + *, + timeout: float | None = None, + cwd: str | None = None, + env: dict[str, str] | None = None, + capture_output: bool = True, + text: bool = True, + **_kwargs: Any, +) -> subprocess.CompletedProcess[Any]: + """Run a CLI in its own process group; kill the **whole group** on timeout/interrupt. + + Node wrappers (e.g. ``codex``) spawn native grandchildren. Plain + ``subprocess.run`` on timeout only kills the parent; the grandchild keeps + stdout open and ``communicate()`` hangs indefinitely. This runner: + + 1. launches with ``start_new_session=True`` (new process group; pgid=pid); + 2. on timeout **or any other exception** (incl. KeyboardInterrupt), + ``os.killpg(pid, SIGKILL)`` the group; + 3. drains pipes with a **bounded** second ``communicate`` (never unbounded). + + Raises ``subprocess.TimeoutExpired`` with attribute + ``killed_process_group=True`` only when killpg actually delivered the signal. + """ + popen_kwargs: dict[str, Any] = { + "cwd": cwd, + "start_new_session": True, + "stdout": subprocess.PIPE if capture_output else None, + "stderr": subprocess.PIPE if capture_output else None, + "text": text, + } + if env is not None: + popen_kwargs["env"] = env + + proc = subprocess.Popen(cmd, **popen_kwargs) # noqa: S603 — eval harness launches user CLIs + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + killed = _kill_process_group(proc) + out, err = _close_pipes_and_reap(proc) + if out is None and err is None: + stdout = _decode_pipe(exc.stdout, text=text) or ("" if text else b"") + stderr = _decode_pipe(exc.stderr, text=text) or ("" if text else b"") + else: + stdout, stderr = out, err + te = subprocess.TimeoutExpired( + cmd=cmd, + timeout=timeout if timeout is not None else 0, + output=stdout, + stderr=stderr, + ) + te.killed_process_group = killed # type: ignore[attr-defined] + raise te from None + except BaseException: + # KeyboardInterrupt / SystemExit / etc. — do not leave the CLI tree running. + # start_new_session means SIGINT no longer reaches the group automatically. + _kill_process_group(proc) + _close_pipes_and_reap(proc) + raise + + return subprocess.CompletedProcess(cmd, proc.returncode if proc.returncode is not None else 0, stdout, stderr) + + +def _note_timeout_kill(notes: list[str], exc: BaseException) -> None: + """Append process-group kill note when killpg actually delivered the signal.""" + if getattr(exc, "killed_process_group", False): + notes.append("timeout_killed_process_group") + + +@dataclass +class AgentRun: + """Normalized result of one agent task execution.""" + + # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} + calls: list[dict[str, Any]] + final_text: str + usage: dict[str, Any] | None + stopped_reason: str + raw_ref: str | None = None + # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens + usage_total: dict[str, Any] | None = None + # Harness extras (optional; defaults keep SDK path simple) + usage_scope: str = "run" # 'run' | 'iteration' + call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'sdk' + hit_max_turns: bool = False + wall_time_s: float = 0.0 + experimental: bool = False + notes: list[str] = field(default_factory=list) + + +class AgentDriver(Protocol): + """Pluggable agent backend for evals.run.""" + + name: str + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: ... + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def strip_mcp_prefix(name: str) -> str: + """Strip Claude/Codex MCP tool name prefixes for classification. + + Examples: + mcp__plane__list_work_items → list_work_items + mcp__plane-mcp-server__find_work_items → find_work_items + """ + if not name: + return name + m = _MCP_PREFIX_RE2.match(name) + if m: + return m.group(1) + return name + + +def is_plane_mcp_tool(name: str) -> bool: + """True when the raw tool name is from our Plane MCP server (pre-strip). + + Claude surfaces MCP tools as ``mcp____``. Our config registers + the server as ``plane``, so names look like ``mcp__plane__find_work_items``. + Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. + """ + if not name: + return False + # mcp__plane__tool or mcp__plane-foo__tool + return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") + + +def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: + """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" + raw = str(name or "") + if not isinstance(args, dict): + args = {"_raw": args} + if is_plane_mcp_tool(raw): + return { + "tool": strip_mcp_prefix(raw), + "args": args, + "origin": "plane", + "raw_tool": raw, + } + return { + "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) + "args": args, + "origin": "client", + "raw_tool": raw, + } + + +def split_plane_and_client_calls( + calls: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Partition tagged calls into plane vs client lists. + + Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls + (SDK path) default to plane so existing harness behavior is unchanged. + """ + plane: list[dict[str, Any]] = [] + client: list[dict[str, Any]] = [] + for c in calls: + origin = c.get("origin") + if origin is None: + raw = str(c.get("raw_tool") or c.get("tool") or "") + if is_plane_mcp_tool(raw): + origin = "plane" + elif raw.startswith("mcp__"): + origin = "client" # other MCP server + else: + origin = "plane" # bare name → assume plane (SDK) + if origin == "client": + client.append(c) + else: + plane.append(c) + return plane, client + + +def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Parse Claude print-mode usage into (raw_usage, usage_total). + + Real envelope (probed 2026-08-12, ``claude -p --output-format json``):: + + { + "usage": { + "input_tokens": 10, # uncached NEW input only — NOT run total + "cache_creation_input_tokens": 17459, + "cache_read_input_tokens": 18464, + "output_tokens": 143, + "iterations": [...], + ... + }, + "modelUsage": { + "": { + "inputTokens": 10, + "outputTokens": 143, + "cacheReadInputTokens": 18464, + "cacheCreationInputTokens": 17459, + "costUSD": 0.037, + ... + } + }, + "total_cost_usd": 0.037 + } + + ``usage.input_tokens`` alone is misleading for multi-turn cached runs (live + rows showed 8–10 while cache_read was 180k+). We keep the split fields and + compute an inclusive total under ``usage_total``; callers must **not** copy + bare ``input_tokens`` into ``cum_input_tokens``. + """ + usage = data.get("usage") + if usage is not None and not isinstance(usage, dict): + usage = None + model_usage = data.get("modelUsage") or data.get("model_usage") + if model_usage is not None and not isinstance(model_usage, dict): + model_usage = None + cost = data.get("total_cost_usd") + if cost is None and isinstance(usage, dict): + cost = usage.get("total_cost_usd") + + if usage is None and model_usage is None and cost is None: + return None, None + + raw = dict(usage or {}) + if cost is not None: + raw["total_cost_usd"] = cost + if model_usage is not None: + raw["modelUsage"] = model_usage + + # Prefer summing modelUsage (per-model run totals) when present + sum_in = sum_out = sum_cr = sum_cc = sum_cost = 0.0 + used_model_usage = False + if model_usage: + for _mid, mu in model_usage.items(): + if not isinstance(mu, dict): + continue + used_model_usage = True + sum_in += float(mu.get("inputTokens") or mu.get("input_tokens") or 0) + sum_out += float(mu.get("outputTokens") or mu.get("output_tokens") or 0) + sum_cr += float(mu.get("cacheReadInputTokens") or mu.get("cache_read_input_tokens") or 0) + sum_cc += float(mu.get("cacheCreationInputTokens") or mu.get("cache_creation_input_tokens") or 0) + sum_cost += float(mu.get("costUSD") or mu.get("cost_usd") or 0) + + if used_model_usage: + uncached_in = int(sum_in) + out_tok = int(sum_out) + cache_read = int(sum_cr) + cache_write = int(sum_cc) + total_cost = float(sum_cost) if sum_cost else cost + else: + uncached_in = int(raw.get("input_tokens") or 0) + out_tok = int(raw.get("output_tokens") or 0) + cache_read = int(raw.get("cache_read_input_tokens") or 0) + cache_write = int(raw.get("cache_creation_input_tokens") or 0) + total_cost = cost + + usage_total: dict[str, Any] = { + "input_tokens": uncached_in, # uncached / new tokens only + "output_tokens": out_tok, + "cache_read_input_tokens": cache_read, + "cache_creation_input_tokens": cache_write, + "total_input_tokens_including_cache": uncached_in + cache_read + cache_write, + "total_cost_usd": total_cost, + "modelUsage": model_usage, + "source": "modelUsage" if used_model_usage else "usage", + } + return raw, usage_total + + +def _claude_project_dir(cwd: Path) -> Path: + """Map a cwd to ``~/.claude/projects/`` (``/`` → ``-``).""" + munged = str(cwd.resolve()).replace("/", "-") + return Path.home() / ".claude" / "projects" / munged + + +def parse_claude_json_result(payload: dict[str, Any] | str) -> dict[str, Any]: + """Extract final text, usage, session id, num_turns from ``claude -p --output-format json``. + + The print-mode JSON envelope is a single object (``type=result``) with + ``result``, ``session_id``, ``num_turns``, ``total_cost_usd``, ``usage``, + and ``modelUsage``. Per-call tool detail is usually **absent** — callers + should fall back to the session transcript. + """ + if isinstance(payload, str): + payload = json.loads(payload) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object from claude, got {type(payload)}") + + data = payload + + final = data.get("result") + if final is None: + final = data.get("final_text") or data.get("text") or "" + if not isinstance(final, str): + final = json.dumps(final, default=str) + + usage, usage_total = normalize_claude_usage(data) + + session_id = data.get("session_id") or data.get("sessionId") + num_turns = data.get("num_turns") + if num_turns is None: + num_turns = data.get("numTurns") + is_error = bool(data.get("is_error") or data.get("isError")) + subtype = data.get("subtype") or "" + stop_reason = data.get("stop_reason") or data.get("terminal_reason") or "" + + # Tool calls rarely present in the result envelope; collect if present. + calls: list[dict[str, Any]] = [] + for key in ("tool_calls", "tools", "calls"): + raw = data.get(key) + if isinstance(raw, list): + for item in raw: + if not isinstance(item, dict): + continue + name = item.get("name") or item.get("tool") or "" + args = item.get("input") or item.get("arguments") or item.get("args") or {} + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + args = {"_raw": args} + calls.append(normalize_tool_call(str(name), args)) + + # Preserve Claude error subtypes (e.g. error_during_execution, error_max_turns). + # is_error alone collapses to "error" and loses the subtype run.py uses for infra_cli. + if is_error and subtype and str(subtype) not in ("success", ""): + stopped = str(subtype) + elif is_error: + stopped = "error" + else: + stopped = str(stop_reason) if stop_reason else "end_turn" + if subtype and subtype not in ("success", "") and stopped == "end_turn": + stopped = str(subtype) + + plane_calls, client_calls = split_plane_and_client_calls(calls) + + return { + "final_text": final, + "usage": usage, + "usage_total": usage_total, + "session_id": session_id, + "num_turns": int(num_turns) if num_turns is not None else None, + "calls": plane_calls, + "client_tool_calls": client_calls, + "stopped_reason": stopped, + "raw": data, + } + + +def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]]: + """Parse ``tool_use`` blocks from a Claude Code session JSONL transcript. + + Returns tagged calls (``origin`` plane|client). Use + ``split_plane_and_client_calls`` before classification. + """ + calls: list[dict[str, Any]] = [] + if not transcript_path.is_file(): + return calls + with transcript_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + msg = row.get("message") if isinstance(row, dict) else None + if not isinstance(msg, dict): + if row.get("type") == "assistant" and isinstance(row.get("content"), list): + content = row["content"] + else: + continue + else: + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") != "tool_use": + continue + name = str(block.get("name") or "") + args = block.get("input") or {} + if not isinstance(args, dict): + args = {"_raw": args} + calls.append(normalize_tool_call(name, args)) + return calls + + +def find_claude_transcript(session_id: str | None, cwd: Path) -> Path | None: + """Locate ``~/.claude/projects//.jsonl``.""" + if not session_id: + return None + candidate = _claude_project_dir(cwd) / f"{session_id}.jsonl" + if candidate.is_file(): + return candidate + # Fallback: scan project dir for a file containing the session id + proj = _claude_project_dir(cwd) + if not proj.is_dir(): + return None + direct = proj / f"{session_id}.jsonl" + if direct.is_file(): + return direct + for p in proj.glob("*.jsonl"): + if session_id in p.name: + return p + return None + + +def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + return {"_raw": raw_args} + return args if isinstance(args, dict) else {"_raw": args} + if isinstance(raw_args, dict): + return raw_args + return {"_raw": raw_args} + + +def parse_codex_jsonl_events(lines: list[str] | str) -> dict[str, Any]: + """Parse ``codex exec --json`` stdout (JSONL) for function_call + final text + usage. + + Supports both schemas: + - **v0.147+ streamable**: ``thread.started`` / ``item.completed`` / ``turn.completed`` + (``thread_id`` matches rollout filename suffix). + - **Legacy**: ``session_meta`` / ``response_item`` / ``event_msg`` payloads. + New keys are tried first; legacy handling is retained. + """ + if isinstance(lines, str): + lines = lines.splitlines() + calls: list[dict[str, Any]] = [] + final_parts: list[str] = [] + usage: dict[str, Any] | None = None + stopped = "end_turn" + session_id: str | None = None + + for line in lines: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + rtype = row.get("type") + + # --- New schema (codex exec --json v0.147+): try first --- + if rtype == "thread.started": + session_id = row.get("thread_id") or session_id + continue + if rtype == "turn.started": + continue + if rtype == "item.completed": + item = row.get("item") if isinstance(row.get("item"), dict) else {} + itype = item.get("type") + if itype == "agent_message": + text = item.get("text") + if text: + final_parts.append(str(text)) + elif itype in ( + "function_call", + "tool_call", + "mcp_tool_call", + "command_execution", + "file_change", + ): + # Proxy sidecar is primary for plane calls; harvest best-effort names. + name = str(item.get("name") or item.get("tool") or item.get("command") or itype) + args = item.get("arguments") or item.get("args") or item.get("input") or {} + calls.append(normalize_tool_call(name, _codex_parse_tool_args(args))) + continue + if rtype == "turn.completed": + u = row.get("usage") if isinstance(row.get("usage"), dict) else {} + if u: + usage = { + "input_tokens": u.get("input_tokens", 0) or 0, + "output_tokens": u.get("output_tokens", 0) or 0, + "cache_read_input_tokens": u.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": u.get("cache_write_input_tokens", 0) or 0, + "total_tokens": u.get("total_tokens"), + } + continue + + # --- Legacy schema --- + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + + if rtype == "session_meta": + session_id = payload.get("id") or session_id + continue + + if rtype == "response_item": + pt = payload.get("type") + if pt == "function_call": + name = str(payload.get("name") or "") + calls.append(normalize_tool_call(name, _codex_parse_tool_args(payload.get("arguments") or "{}"))) + elif pt == "message": + # Assistant final-ish content + role = payload.get("role") + content = payload.get("content") + if role == "assistant" and isinstance(content, list): + for c in content: + if isinstance(c, dict) and c.get("type") in ("output_text", "text"): + t = c.get("text") or c.get("output_text") + if t: + final_parts.append(str(t)) + continue + + if rtype == "event_msg": + pt = payload.get("type") + if pt == "agent_message": + msg = payload.get("message") or payload.get("text") + if msg: + final_parts.append(str(msg)) + elif pt == "token_count": + info = payload.get("info") or {} + total = info.get("total_token_usage") or info.get("last_token_usage") or {} + if isinstance(total, dict): + usage = { + "input_tokens": total.get("input_tokens", 0) or 0, + "output_tokens": total.get("output_tokens", 0) or 0, + "cache_read_input_tokens": total.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": total.get("cache_write_input_tokens", 0) or 0, + "total_tokens": total.get("total_tokens"), + } + elif pt == "task_complete": + stopped = "end_turn" + elif pt == "turn_aborted": + stopped = "aborted" + continue + + plane_calls, client_calls = split_plane_and_client_calls(calls) + return { + "calls": plane_calls, + "client_tool_calls": client_calls, + "final_text": "\n".join(final_parts).strip(), + "usage": usage, + "stopped_reason": stopped, + "session_id": session_id, + } + + +def parse_codex_rollout_calls(rollout_path: Path) -> list[dict[str, Any]]: + """Parse function_call records from a Codex session rollout JSONL (plane only).""" + if not rollout_path.is_file(): + return [] + lines = rollout_path.read_text(encoding="utf-8").splitlines() + return parse_codex_jsonl_events(lines)["calls"] + + +def find_codex_rollout(session_id: str | None, *, after_ts: float | None = None) -> Path | None: + """Find a rollout JSONL under ``~/.codex/sessions`` matching *session_id* exactly. + + Matches filename containing the id (rollout filenames end with ``thread_id``) + or a first-line ``session_meta`` / ``thread.started`` id field. + + **No newest-after-ts fallback**: under parallel runs that would pick another + task's rollout and corrupt final_text. Callers should note + ``codex_rollout_unmatched`` when this returns None. + + ``after_ts`` is accepted for API compatibility but ignored. + """ + del after_ts # intentionally unused — see docstring + if not session_id: + return None + root = Path.home() / ".codex" / "sessions" + if not root.is_dir(): + return None + sid = str(session_id) + for p in root.rglob("*.jsonl"): + if sid in p.name: + return p + for p in root.rglob("rollout-*.jsonl"): + try: + with p.open(encoding="utf-8") as fh: + first = fh.readline() + row = json.loads(first) + except Exception: + continue + # Legacy session_meta + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + if payload.get("id") == sid: + return p + # New thread.started on first line (unusual but possible) + if row.get("type") == "thread.started" and row.get("thread_id") == sid: + return p + if row.get("thread_id") == sid or row.get("session_id") == sid: + return p + return None + + +def write_claude_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write a Claude-compatible mcp-config JSON file.""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def write_codex_mcp_override_args( + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> list[str]: + """Build ``codex exec -c ...`` overrides for a stdio MCP server. + + Codex stores MCP under ``[mcp_servers.]`` with ``command``, ``args``, + and ``env`` (see user config.toml). Overrides use dotted ``-c`` paths. + """ + out: list[str] = [ + "-c", + f"mcp_servers.{server_name}.command={json.dumps(command)}", + "-c", + f"mcp_servers.{server_name}.args={json.dumps(args)}", + ] + # env table — pass each key + for k, v in env.items(): + out.extend(["-c", f"mcp_servers.{server_name}.env.{k}={json.dumps(v)}"]) + return out + + +# --------------------------------------------------------------------------- +# Claude CLI driver +# --------------------------------------------------------------------------- + + +class ClaudeCliDriver: + """Run tasks via ``claude -p`` on the user's Claude Code subscription.""" + + name = "claude-cli" + + def __init__( + self, + *, + claude_bin: str = "claude", + python_bin: str | None = None, + permission_mode: str = "bypassPermissions", + strict_mcp: bool = True, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.claude_bin = claude_bin + self.python_bin = python_bin or sys.executable + self.permission_mode = permission_mode + self.strict_mcp = strict_mcp + self._runner = runner or run_cli_subprocess + # Full replacement for the MCP server launch (external surfaces under + # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = [] + t0 = time.perf_counter() + + with tempfile.TemporaryDirectory(prefix="plane-eval-claude-") as td: + td_path = Path(td) + mcp_cfg = td_path / "mcp.json" + sidecar = td_path / "proxy-sidecar.jsonl" + # Only pass Plane-related env into the MCP child (plus PATH/HOME if present). + child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env = ensure_proxy_pythonpath(child_env) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + write_claude_mcp_config( + mcp_cfg, + command=server_cmd, + args=server_args, + env=child_env, + server_name="plane", + ) + + cmd: list[str] = [ + self.claude_bin, + "-p", + "--output-format", + "json", + "--mcp-config", + str(mcp_cfg), + "--permission-mode", + self.permission_mode, + "--max-turns", + str(max_turns), + ] + if self.strict_mcp: + cmd.append("--strict-mcp-config") + if model: + cmd.extend(["--model", model]) + if system: + cmd.extend(["--append-system-prompt", system]) + # Allow MCP tools from our server without interactive prompts + # --allowedTools is variadic and would swallow the trailing prompt; use = form. + cmd.append("--allowedTools=mcp__plane__*") + cmd.append(prompt) + + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + _note_timeout_kill(notes, exc) + # Wait for proxy finalization before harvesting / temp dir teardown. + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = harvest_proxy_after_cli_timeout( + calls, client_calls, sidecar, notes + ) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + + parsed: dict[str, Any] | None = None + parse_err: str | None = None + # JSON may be the whole stdout or the last JSON object line + for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): + if not candidate or not candidate.lstrip().startswith("{"): + continue + try: + parsed = parse_claude_json_result(candidate) + break + except (json.JSONDecodeError, ValueError, TypeError) as exc: + parse_err = str(exc) + continue + + if parsed is None: + notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + if self.use_proxy: + apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise RuntimeError(f"claude cli failed: {detail}") + + # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + # JSON rarely embeds per-call tool detail — prefer transcript when present. + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "json" if (calls or client_calls) else "json" + session_id = parsed.get("session_id") + transcript = find_claude_transcript(session_id, cwd) + if transcript is not None: + tagged = parse_claude_transcript_calls(transcript) + t_plane, t_client = split_plane_and_client_calls(tagged) + if t_plane or t_client: + calls, client_calls = t_plane, t_client + call_source = "transcript" + notes.append(f"calls_from_transcript:{transcript}") + if not calls and not client_calls: + notes.append("no_tool_calls_in_json_or_transcript") + + # Proxy sidecar (when enabled) replaces CLI-parsed plane calls. + if self.use_proxy: + calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proxy_src == "proxy": + call_source = "proxy" + + num_turns = parsed.get("num_turns") + hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) + stopped = parsed["stopped_reason"] + if hit_max and stopped in ("end_turn", "completed", ""): + stopped = "max_turns" + + raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=parsed["final_text"], + usage=parsed.get("usage"), + usage_total=parsed.get("usage_total"), + stopped_reason=stopped, + raw_ref=raw_ref, + usage_scope="run", + call_source=call_source, + hit_max_turns=hit_max, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +# --------------------------------------------------------------------------- +# Codex CLI driver (experimental — do not spend live quota from CI) +# --------------------------------------------------------------------------- + + +class CodexCliDriver: + """Run tasks via ``codex exec`` (experimental; metered quota). + + Live invocation is supported for the interface, but the eval harness should + only exercise this driver when the team explicitly opts in. Offline tests + inject a fake runner and never touch the real binary. + """ + + name = "codex-cli" + experimental = True + + def __init__( + self, + *, + codex_bin: str = "codex", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + allow_live: bool = False, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.codex_bin = codex_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.allow_live = allow_live + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes = ["experimental:codex-cli"] + if self._runner is run_cli_subprocess and not self.allow_live: + raise RuntimeError( + "CodexCliDriver refuses live runs by default (metered weekly quota). " + "Pass allow_live=True or inject a fake runner for tests." + ) + + child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + with tempfile.TemporaryDirectory(prefix="plane-eval-codex-") as td: + td_path = Path(td) + sidecar = td_path / "proxy-sidecar.jsonl" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env = ensure_proxy_pythonpath(child_env) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + mcp_args = write_codex_mcp_override_args( + command=server_cmd, + args=server_args, + env=child_env, + server_name="plane", + ) + cmd: list[str] = [ + self.codex_bin, + "exec", + "--json", + "--skip-git-repo-check", + *mcp_args, + ] + if model: + cmd.extend(["-m", model]) + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd.append(full_prompt) + + t0 = time.perf_counter() + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + _note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "stream" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + experimental=True, + notes=notes, + ) + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + parsed = parse_codex_jsonl_events(stdout) + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "stream" + session_id = parsed.get("session_id") + + # Exact-match rollout only — never steal another parallel task's file. + need_rollout = (not calls and not client_calls) or not parsed.get("final_text") + if need_rollout and session_id: + rollout = find_codex_rollout(session_id) + if rollout is not None: + full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) + if not calls and not client_calls: + calls = list(full.get("calls") or []) + client_calls = list(full.get("client_tool_calls") or []) + if calls or client_calls: + call_source = "transcript" + notes.append(f"calls_from_rollout:{rollout}") + if full.get("final_text") and not parsed.get("final_text"): + parsed["final_text"] = full["final_text"] + notes.append(f"final_text_from_rollout:{rollout}") + if full.get("usage") and not parsed.get("usage"): + parsed["usage"] = full["usage"] + else: + notes.append("codex_rollout_unmatched") + elif need_rollout and not session_id: + notes.append("codex_rollout_unmatched") + + if self.use_proxy: + calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proxy_src == "proxy": + call_source = "proxy" + + if proc.returncode != 0: + notes.append(f"codex_exit={proc.returncode}") + + usage = parsed.get("usage") + usage_total = None + if isinstance(usage, dict): + usage_total = { + "input_tokens": usage.get("input_tokens"), + "output_tokens": usage.get("output_tokens"), + "cache_read_input_tokens": usage.get("cache_read_input_tokens"), + "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), + "total_input_tokens_including_cache": ( + int(usage.get("input_tokens") or 0) + + int(usage.get("cache_read_input_tokens") or 0) + + int(usage.get("cache_creation_input_tokens") or 0) + ), + "source": "codex_token_count", + } + + raw_ref = f"session:{session_id}" if session_id else None + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=parsed.get("final_text") or "", + usage=usage, + usage_total=usage_total, + stopped_reason=parsed.get("stopped_reason") or "end_turn", + raw_ref=raw_ref, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, # codex exec has no max-turns flag in --help + wall_time_s=round(wall, 3), + experimental=True, + notes=notes, + ) + + +# --------------------------------------------------------------------------- +# Antigravity CLI (agy) — proxy-first +# --------------------------------------------------------------------------- + + +def write_antigravity_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write ``mcpServers`` map JSON (Antigravity / agy mcp_config shape).""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def prepare_antigravity_fake_home( + fake_home: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + real_home: Path | None = None, +) -> None: + """Build an isolated HOME for agy with MCP config + shared auth artifacts. + + Writes mcp_config.json to BOTH documented locations (cheap; live probe + should settle which path agy actually reads): + - ~/.gemini/config/mcp_config.json + - ~/.gemini/antigravity-cli/mcp_config.json + + Creates ``antigravity-cli`` as a **real directory** (never a symlink of the + whole tree — that would write mcp_config and runtime logs into real user + state). Auth artifacts (``antigravity-oauth-token``) are plain **copies** — + never symlinks — so an in-place token refresh cannot write through into the + real home. Staleness over a single eval run is negligible. + """ + real_home = real_home or Path.home() + gemini_root = fake_home / ".gemini" + gemini_root.mkdir(parents=True, exist_ok=True) + + real_cli = real_home / ".gemini" / "antigravity-cli" + fake_cli = gemini_root / "antigravity-cli" + # Always a real directory — never symlink the whole tree. + if fake_cli.is_symlink() or fake_cli.is_file(): + fake_cli.unlink() + fake_cli.mkdir(parents=True, exist_ok=True) + + # Share auth via plain COPY only — never symlink (in-place token refresh + # must not write through into the real home). + if real_cli.is_dir(): + for name in ("antigravity-oauth-token",): + src = real_cli / name + dst = fake_cli / name + if src.is_file() and not dst.exists(): + try: + dst.write_bytes(src.read_bytes()) + except OSError: + pass + + # Dual write as real files (not through any symlink). + for rel in ( + Path(".gemini") / "config" / "mcp_config.json", + Path(".gemini") / "antigravity-cli" / "mcp_config.json", + ): + write_antigravity_mcp_config( + fake_home / rel, + command=command, + args=args, + env=env, + server_name="plane", + ) + + +class AntigravityCliDriver: + """Run tasks via Google Antigravity CLI (``agy``). + + Probed flags (2026-08-12, ``agy --help``): + - ``-p`` / ``--print`` headless single-prompt mode + - ``--output-format`` text|json|stream-json + - ``--model``, ``--dangerously-skip-permissions`` + - MCP via ``~/.gemini/config/mcp_config.json`` (``mcpServers`` map; + stdio: command/args/env). No CLI flag for MCP config → HOME isolation. + - No max-turns / turn-cap flag in help → ``hit_max_turns=False`` + note. + + Tool calls come from the recording proxy sidecar (protocol-layer), not + agy stdout parsing. + """ + + name = "antigravity-cli" + + def __init__( + self, + *, + agy_bin: str = "agy", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.agy_bin = agy_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = ["no_turn_cap"] + t0 = time.perf_counter() + child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + + with tempfile.TemporaryDirectory(prefix="plane-eval-antigravity-") as td: + td_path = Path(td) + sidecar = td_path / "proxy-sidecar.jsonl" + fake_home = td_path / "home" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env_plane = ensure_proxy_pythonpath(child_env_plane) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + prepare_antigravity_fake_home( + fake_home, + command=server_cmd, + args=server_args, + env=child_env_plane, + ) + + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd: list[str] = [ + self.agy_bin, + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + ] + if model: + cmd.extend(["--model", model]) + cmd.append(full_prompt) + + run_env = {**os.environ, "HOME": str(fake_home)} + if "PATH" in child_env_plane: + run_env["PATH"] = child_env_plane["PATH"] + + timeout_s = max(120, max_turns * 60) + try: + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + env=run_env, + ) + except TypeError: + # Some test runners reject ``env=``; retry without it. + # TimeoutExpired from this path must still hit the harvest below. + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + _note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + usage_scope="run", + call_source=call_source, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proc.returncode != 0: + notes.append(f"agy_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + final_text = stdout.strip() + try: + if final_text.lstrip().startswith("{"): + blob = json.loads(final_text) + if isinstance(blob, dict): + final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) + except json.JSONDecodeError: + pass + + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=final_text, + usage=None, + stopped_reason="error" if proc.returncode else "end_turn", + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +# --------------------------------------------------------------------------- +# OpenCode CLI — proxy-first +# --------------------------------------------------------------------------- + + +def write_opencode_mcp_config( + path: Path, + *, + command: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write project ``opencode.json`` with a local MCP server entry. + + Schema (opencode.ai docs / probed binary strings, 2026-08-12):: + + {"mcp": {"plane": {"type": "local", "command": [...], "environment": {...}}}} + """ + cfg = { + "$schema": "https://opencode.ai/config.json", + "mcp": { + server_name: { + "type": "local", + "command": list(command), + "environment": env, + "enabled": True, + } + }, + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +class OpencodeCliDriver: + """Run tasks via ``opencode run`` (proxy-first call recording). + + Probed flags (2026-08-12): + - ``opencode run [message..]`` non-interactive + - ``--format json|default``, ``-m/--model`` + - MCP via ``opencode.json`` ``mcp`` section (local: type/command/environment) + written into the task cwd (or a temp project dir). + - No turn-cap flag → ``hit_max_turns=False`` + note ``no_turn_cap``. + """ + + name = "opencode-cli" + + def __init__( + self, + *, + opencode_bin: str = "opencode", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.opencode_bin = opencode_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + base_cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = ["no_turn_cap"] + t0 = time.perf_counter() + child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + + with tempfile.TemporaryDirectory(prefix="plane-eval-opencode-", dir=str(base_cwd)) as td: + # Project-local opencode.json so we do not pollute the user's global config. + proj = Path(td) + sidecar = proj / "proxy-sidecar.jsonl" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + launch = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + child_env_plane = ensure_proxy_pythonpath(child_env_plane) + else: + launch = real_cmd + write_opencode_mcp_config( + proj / "opencode.json", + command=launch, + env=child_env_plane, + server_name="plane", + ) + + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd: list[str] = [ + self.opencode_bin, + "run", + "--format", + "json", + ] + if model: + cmd.extend(["-m", model]) + cmd.append(full_prompt) + + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(proj), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + _note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + usage_scope="run", + call_source=call_source, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proc.returncode != 0: + notes.append(f"opencode_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + final_text = stdout.strip() + # JSONL events: concatenate text-ish fields best-effort. + if final_text and "\n" in final_text: + parts: list[str] = [] + for line in final_text.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + for key in ("text", "message", "part", "delta"): + v = row.get(key) + if isinstance(v, str) and v.strip(): + parts.append(v) + if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): + parts.append(row["content"]) + if parts: + final_text = "\n".join(parts) + + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=final_text, + usage=None, + stopped_reason="error" if proc.returncode else "end_turn", + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + +KNOWN_DRIVERS = frozenset({"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) + + +def get_driver(name: str, **kwargs: Any) -> AgentDriver | None: + """Return a driver instance, or None for the in-process ``sdk`` path.""" + key = (name or "sdk").strip().lower() + if key == "sdk": + return None # handled inline in evals.run + if key == "claude-cli": + return ClaudeCliDriver(**kwargs) + if key == "codex-cli": + return CodexCliDriver(**kwargs) + if key == "antigravity-cli": + return AntigravityCliDriver(**kwargs) + if key == "opencode-cli": + return OpencodeCliDriver(**kwargs) + raise ValueError(f"unknown driver {name!r}; expected one of {sorted(KNOWN_DRIVERS)}") + + +def agent_run_to_harness_dict( + run: AgentRun, + *, + optimal: set[str], + alternate: set[str], + classify: Callable[[str, set[str], set[str]], str], + skip_result_tokens: bool = True, +) -> dict[str, Any]: + """Map an ``AgentRun`` onto the dict shape expected by ``run_live`` rows. + + Only **plane** MCP tools are classified and counted in ``num_calls`` / + mispick metrics. Client built-ins (``ToolSearch``, …) go to + ``client_tool_calls`` and are excluded. + + CLI drivers never populate ``cum_input_tokens`` from bare + ``usage.input_tokens`` (that field is uncached-only under Claude Code and + misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. + """ + # Re-split in case callers passed a mixed list + plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) + client_src = list(run.client_tool_calls) + client_extra + + calls: list[dict[str, Any]] = [] + for c in plane_src: + tool = c.get("tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + rec: dict[str, Any] = { + "tool": tool, + "class": classify(str(tool), optimal, alternate), + "args_chars": args_chars, + "result_tokens": None, + "result_chars": int(c["result_chars"]) if c.get("result_chars") is not None else 0, + "result_kind": "text", + "is_error": bool(c.get("is_error")), + } + if c.get("duration_ms") is not None: + rec["duration_ms"] = c["duration_ms"] + # Action-dispatch surfaces: the action arg IS the second half of the + # tool choice — keep it (args content is otherwise not persisted). + if isinstance(args, dict) and isinstance(args.get("action"), str): + rec["action"] = args["action"] + if skip_result_tokens: + rec["result_tokens_skipped"] = "no API key / CLI driver has no count_tokens" + calls.append(rec) + + client_tool_calls: list[dict[str, Any]] = [] + for c in client_src: + tool = c.get("tool") or c.get("raw_tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + client_tool_calls.append( + { + "tool": tool, + "args_chars": args_chars, + "raw_tool": c.get("raw_tool") or tool, + } + ) + + stop_reason = run.stopped_reason + hit_max = run.hit_max_turns + if hit_max: + stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" + + errored = sum(1 for c in calls if c.get("is_error")) + alternate_n = sum(1 for c in calls if c["class"] == "alternate") + out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") + + # CLI path: never write misleading cum_input_tokens from uncached-only field + is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" + usage_total = run.usage_total + if usage_total is None and isinstance(run.usage, dict) and is_cli: + # Best-effort rebuild if driver forgot usage_total + _, usage_total = normalize_claude_usage({"usage": run.usage, "modelUsage": run.usage.get("modelUsage")}) + + if is_cli and skip_result_tokens: + cum_input: int | None = None + cum_reason: str | None = ( + "CLI driver: Claude usage.input_tokens is uncached-only; " + "see usage_total (cache_read/cache_creation/output/cost) for run accounting" + ) + usage_per_iteration: list[dict[str, int]] = [] + else: + cum_input = 0 + cum_reason = None + usage_per_iteration = [] + if run.usage and run.usage_scope == "iteration": + pass # SDK fills this separately + + return { + "final_text": run.final_text, + "calls": calls, + "num_calls": len(calls), + "client_tool_calls": client_tool_calls, + "client_tool_call_count": len(client_tool_calls), + "errored_calls": errored, + "alternate_calls": alternate_n, + "out_of_set_calls": out_of_set_n, + "total_result_tokens": 0 + if skip_result_tokens + else sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None), + "usage_per_iteration": usage_per_iteration, + "cum_input_tokens": cum_input, + "cum_input_tokens_reason": cum_reason, + "wall_time_s": run.wall_time_s, + "stop_reason": stop_reason, + "hit_max_iterations": hit_max, + "result_pair_mismatch": False, + "token_count_failures": 0, + "usage_scope": run.usage_scope, + "call_source": run.call_source, + "driver_raw_ref": run.raw_ref, + "driver_notes": list(run.notes), + "result_tokens_skipped_reason": ( + "CLI driver: count_tokens requires Anthropic API key; skipped" if skip_result_tokens else None + ), + "usage": run.usage, + "usage_total": usage_total, + } diff --git a/evals/env.sh b/evals/env.sh new file mode 100755 index 00000000..1a895c55 --- /dev/null +++ b/evals/env.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +# Local eval environment bootstrap: plane-ee API + mock feature-flag server. +# +# Required env (no defaults): +# PLANE_EE_API_DIR — path to plane-ee apps/api (or monorepo root with apps/api) +# PLANE_EE_VENV — path to the Python venv used to run plane-ee manage.py +# +# Mock flags are launched via the *repo* venv (REPO/.venv/bin/python -m evals.mock_flags), +# not PLANE_EE_VENV — that venv only runs plane-ee. Both FEATURE_FLAG_SERVER_BASE_URL +# and health checks use http://127.0.0.1:9911 (mock binds 127.0.0.1). +# +# Usage: +# evals/env.sh up # start API :8000 + mock flags :9911; write evals/.env-pids +# evals/env.sh down # kill PIDs from evals/.env-pids (identity-checked) +# evals/env.sh status # report liveness +# +# API is launched with --noreload, API_KEY_RATE_LIMIT=5000/min, and +# FEATURE_FLAG_SERVER_BASE_URL=http://127.0.0.1:9911. Sources $PLANE_EE_API_DIR/.env +# (or apps/api/.env) with set -a. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PID_FILE="$SCRIPT_DIR/.env-pids" +API_PORT=8000 +FLAG_PORT=9911 +FLAG_URL="http://127.0.0.1:${FLAG_PORT}" +API_URL="http://127.0.0.1:${API_PORT}" +FLAG_BASE_URL="http://127.0.0.1:${FLAG_PORT}" +# Repo venv used for evals.mock_flags (parameterized relative to this script). +MOCK_FLAGS_PYTHON="${REPO_ROOT}/.venv/bin/python" + +die() { echo "error: $*" >&2; exit 1; } + +require_env() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + die "$name is required (no default)" + fi +} + +resolve_api_dir() { + require_env PLANE_EE_API_DIR + require_env PLANE_EE_VENV + local base="${PLANE_EE_API_DIR}" + if [[ -d "$base/apps/api" ]]; then + echo "$base/apps/api" + elif [[ -d "$base" ]]; then + echo "$base" + else + die "PLANE_EE_API_DIR not a directory: $base" + fi +} + +resolve_python() { + local venv="${PLANE_EE_VENV}" + if [[ -x "$venv/bin/python" ]]; then + echo "$venv/bin/python" + elif [[ -x "$venv" ]]; then + echo "$venv" + else + die "PLANE_EE_VENV has no bin/python: $venv" + fi +} + +# True if $1 is a live pid whose command line looks like our managed process. +pid_is_ours() { + local pid="$1" + local kind="$2" # flag | api + if ! kill -0 "$pid" 2>/dev/null; then + return 1 + fi + local cmd + cmd="$(ps -o command= -p "$pid" 2>/dev/null || true)" + if [[ -z "$cmd" ]]; then + return 1 + fi + case "$kind" in + flag) + [[ "$cmd" == *evals.mock_flags* ]] || [[ "$cmd" == *mock_flags* ]] + ;; + api) + [[ "$cmd" == *manage.py*runserver* ]] || [[ "$cmd" == *"manage.py"*"runserver"* ]] + ;; + *) + return 1 + ;; + esac +} + +cmd_up() { + local api_dir py + api_dir="$(resolve_api_dir)" + py="$(resolve_python)" + + if [[ ! -x "$MOCK_FLAGS_PYTHON" ]]; then + die "mock-flags python not found/executable: $MOCK_FLAGS_PYTHON (create repo .venv)" + fi + + if [[ -f "$PID_FILE" ]]; then + # shellcheck disable=SC1090 + source "$PID_FILE" + local live=0 + if [[ -n "${flag_pid:-}" ]] && kill -0 "$flag_pid" 2>/dev/null; then + live=1 + fi + if [[ -n "${api_pid:-}" ]] && kill -0 "$api_pid" 2>/dev/null; then + live=1 + fi + if [[ $live -eq 1 ]]; then + die "already up (run 'down' first) — pidfile $PID_FILE has live process(es)" + fi + echo "warning: stale pidfile $PID_FILE (no live pids); replacing" >&2 + rm -f "$PID_FILE" + fi + + # Mock flag server first (API may call it on boot). + local flag_log="$SCRIPT_DIR/.mock_flags.log" + ( + cd "$REPO_ROOT" + export PLANE_EE_API_DIR + exec "$MOCK_FLAGS_PYTHON" -m evals.mock_flags "$FLAG_PORT" + ) >"$flag_log" 2>&1 & + local flag_pid=$! + echo "started mock_flags pid=$flag_pid log=$flag_log" + # Record flag_pid immediately so a later failure does not strand :9911. + { + echo "flag_pid=$flag_pid" + echo "flag_port=$FLAG_PORT" + } >"$PID_FILE" + + # Source plane-ee .env + local env_file="" + if [[ -f "$api_dir/.env" ]]; then + env_file="$api_dir/.env" + elif [[ -f "$PLANE_EE_API_DIR/.env" ]]; then + env_file="$PLANE_EE_API_DIR/.env" + fi + if [[ -n "$env_file" ]]; then + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a + echo "sourced $env_file" + else + echo "warning: no .env found under $api_dir or $PLANE_EE_API_DIR" >&2 + fi + + local api_log="$SCRIPT_DIR/.api_runserver.log" + ( + cd "$api_dir" + export API_KEY_RATE_LIMIT="5000/min" + export FEATURE_FLAG_SERVER_BASE_URL="${FLAG_BASE_URL}" + # --noreload: $! must be the real server, not the autoreloader parent. + exec "$py" manage.py runserver --noreload "0.0.0.0:${API_PORT}" + ) >"$api_log" 2>&1 & + local api_pid=$! + echo "started api runserver pid=$api_pid log=$api_log" + { + echo "flag_pid=$flag_pid" + echo "api_pid=$api_pid" + echo "flag_port=$FLAG_PORT" + echo "api_port=$API_PORT" + } >"$PID_FILE" + + # Health checks (retry briefly) + local i + for i in 1 2 3 4 5 6 7 8 9 10; do + if curl -sf -o /dev/null -X POST "$FLAG_URL/api/feature-flags/" \ + -H 'Content-Type: application/json' -d '{}'; then + echo "health: mock flags :$FLAG_PORT OK" + break + fi + if [[ $i -eq 10 ]]; then + die "mock flags health check failed on :$FLAG_PORT (see $flag_log)" + fi + sleep 0.5 + done + + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do + if curl -sf -o /dev/null "$API_URL/" || curl -sf -o /dev/null "$API_URL/api/"; then + echo "health: api :$API_PORT OK" + break + fi + # Accept any HTTP response as "up" (auth redirects still mean listening). + if curl -s -o /dev/null -w "%{http_code}" "$API_URL/" | grep -qE '^[2345]'; then + echo "health: api :$API_PORT listening" + break + fi + if [[ $i -eq 30 ]]; then + die "api health check failed on :$API_PORT (see $api_log)" + fi + sleep 1 + done + + echo "env up: pids in $PID_FILE" +} + +cmd_down() { + if [[ ! -f "$PID_FILE" ]]; then + echo "no $PID_FILE — nothing to stop" + return 0 + fi + # shellcheck disable=SC1090 + source "$PID_FILE" + for name_kind in flag_pid:flag api_pid:api; do + local name="${name_kind%%:*}" + local kind="${name_kind##*:}" + local pid="${!name:-}" + if [[ -z "$pid" ]]; then + echo "$name=unset" + continue + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name=$pid not running" + continue + fi + if ! pid_is_ours "$pid" "$kind"; then + echo "warning: refusing to kill $name=$pid — command line is not a managed evals process" >&2 + ps -o command= -p "$pid" 2>/dev/null | sed 's/^/ cmd: /' >&2 || true + continue + fi + kill "$pid" 2>/dev/null || true + sleep 0.3 + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" 2>/dev/null || true + fi + echo "stopped $name=$pid" + done + rm -f "$PID_FILE" + echo "env down" +} + +cmd_status() { + local flag_ok=0 api_ok=0 + if curl -sf -o /dev/null -X POST "$FLAG_URL/api/feature-flags/" \ + -H 'Content-Type: application/json' -d '{}' 2>/dev/null; then + flag_ok=1 + fi + local code + code="$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/" 2>/dev/null || echo 000)" + if [[ "$code" =~ ^[2345] ]]; then + api_ok=1 + fi + echo "mock_flags :$FLAG_PORT $([[ $flag_ok -eq 1 ]] && echo UP || echo DOWN)" + echo "api :$API_PORT $([[ $api_ok -eq 1 ]] && echo UP || echo DOWN) (http $code)" + if [[ -f "$PID_FILE" ]]; then + echo "pid file: $PID_FILE" + cat "$PID_FILE" + else + echo "pid file: (none)" + fi + [[ $flag_ok -eq 1 && $api_ok -eq 1 ]] +} + +usage() { + cat < dict[str, Any]: + """Full wire-shaped tool dict for token counting (includes outputSchema when present).""" + name = getattr(tool, "name", None) or (tool.get("name") if isinstance(tool, dict) else "") or "" + description = (getattr(tool, "description", None) if not isinstance(tool, dict) else tool.get("description")) or "" + input_schema = ( + getattr(tool, "inputSchema", None) + if not isinstance(tool, dict) + else (tool.get("inputSchema") or tool.get("input_schema")) + ) or {} + output_schema = ( + getattr(tool, "outputSchema", None) + if not isinstance(tool, dict) + else (tool.get("outputSchema") or tool.get("output_schema")) + ) + d: dict[str, Any] = { + "name": name, + "description": description, + "input_schema": input_schema, + } + if output_schema is not None: + d["output_schema"] = output_schema + return d + + +def tool_payload_model_facing(tool: Any) -> dict[str, Any]: + """Model-facing payload: name + description + input_schema only (no outputSchema).""" + wire = tool_payload_wire(tool) + return { + "name": wire["name"], + "description": wire["description"], + "input_schema": wire["input_schema"], + } + + +def count_tool_tokens( + tools: list[Any], + *, + encode: Any | None = None, +) -> tuple[list[ToolTokenRow], int, int]: + """Count cl100k tokens per tool for wire and model-facing serializations. + + ``encode`` is a callable ``str -> list[int]`` (tiktoken Encoding.encode). When + None, imports tiktoken cl100k_base. Returns (per-tool rows sorted by wire + tokens desc, total_wire, total_model_facing). + """ + if encode is None: + import tiktoken + + enc = tiktoken.get_encoding("cl100k_base") + encode = enc.encode + + rows: list[ToolTokenRow] = [] + total_wire = 0 + total_model = 0 + for t in tools: + wire = tool_payload_wire(t) + model = tool_payload_model_facing(t) + w_tok = len(encode(json.dumps(wire, separators=(",", ":"), ensure_ascii=False))) + m_tok = len(encode(json.dumps(model, separators=(",", ":"), ensure_ascii=False))) + has_out = "output_schema" in wire and wire["output_schema"] is not None + rows.append( + ToolTokenRow( + name=str(wire["name"]), + wire_tokens=w_tok, + model_facing_tokens=m_tok, + has_output_schema=has_out, + ) + ) + total_wire += w_tok + total_model += m_tok + rows.sort(key=lambda r: r.wire_tokens, reverse=True) + return rows, total_wire, total_model + + +def _listing_stdio_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from EVAL_* credentials via the shared run.py helper.""" + if not os.environ.get("EVAL_PLANE_API_KEY") or not os.environ.get("EVAL_PLANE_WORKSPACE_SLUG"): + raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for listing measurement") + return stdio_server_env(surface=surface, extra=extra) + + +async def list_tools_from_stdio( + command: str, + args: list[str], + env: dict[str, str], +) -> list[Any]: + """Connect to a stdio MCP server and return all tools (paginated).""" + from mcp import ClientSession + from mcp.client.stdio import StdioServerParameters, stdio_client + + params = StdioServerParameters(command=command, args=args, env=env) + tools: list[Any] = [] + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + cursor = None + while True: + page = await session.list_tools(cursor=cursor) if cursor else await session.list_tools() + tools.extend(page.tools or []) + cursor = getattr(page, "nextCursor", None) or getattr(page, "next_cursor", None) + if not cursor: + break + return tools + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Measure MCP tool listing tokens (cl100k)") + p.add_argument( + "--surface", + type=str, + default=None, + help=( + "Tool surface: full | v2 | v2-schema. Default full when not using --server-cmd; " + "with --server-cmd, default label is 'external' (or pass a free-form label)." + ), + ) + p.add_argument( + "--server-cmd", + type=str, + default=None, + help="External MCP stdio launch command (shlex-split); free-form surface label", + ) + p.add_argument( + "--server-env", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra env for the MCP server child; repeatable", + ) + p.add_argument("--top", type=int, default=10, help="Top-N tools by wire tokens (default 10)") + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + extra: dict[str, str] = {} + for pair in args.server_env: + key, sep, val = pair.partition("=") + if not sep or not key: + print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) + return 2 + extra[key] = val + + if args.server_cmd: + parts = shlex.split(args.server_cmd) + if not parts: + print("error: --server-cmd is empty", file=sys.stderr) + return 2 + command, cmd_args = parts[0], parts[1:] + # Never label external runs as "full" — default is "external". + surface_label = (args.surface or "external").strip() or "external" + try: + # surface="full" leaves PLANE_MCP_SURFACE unset; extra may set foreign vars. + env = _listing_stdio_env(surface="full", extra=extra or None) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + else: + surface = (args.surface or "full").strip().lower() + if surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", + file=sys.stderr, + ) + return 2 + command = sys.executable + cmd_args = ["-m", "plane_mcp", "stdio"] + surface_label = surface + try: + env = _listing_stdio_env(surface=surface, extra=extra or None) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + try: + tools = asyncio.run(list_tools_from_stdio(command, cmd_args, env)) + except Exception as exc: + print(f"error: failed to list tools: {exc}", file=sys.stderr) + return 1 + + try: + rows, total_wire, total_model = count_tool_tokens(tools) + except ImportError: + print( + "error: tiktoken is required (install with: uv pip install '.[dev]')", + file=sys.stderr, + ) + return 1 + + with_out = sum(1 for r in rows if r.has_output_schema) + print( + f"surface={surface_label} tools={len(rows)} " + f"listing_tokens_cl100k={total_wire} " + f"model_facing(no_outputSchema)={total_model} " + f"tools_with_outputSchema={with_out}" + ) + top_n = max(0, int(args.top)) + if top_n and rows: + print(f"top {min(top_n, len(rows))} by wire tokens:") + for r in rows[:top_n]: + flag = " +out" if r.has_output_schema else "" + print(f" {r.wire_tokens:6d} {r.name}{flag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/mock_flags.py b/evals/mock_flags.py new file mode 100644 index 00000000..37300b96 --- /dev/null +++ b/evals/mock_flags.py @@ -0,0 +1,120 @@ +"""All-flags-on mock of the disco feature-flag service (local eval testing only). + +POST /api/feature-flags/ -> {"values": {: true}} + +Requires PLANE_EE_API_DIR (path to plane-ee apps/api, or the monorepo root +containing apps/api/plane/payment/flags/flag.py). +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +def _resolve_flag_module_path() -> Path: + base = os.environ.get("PLANE_EE_API_DIR", "").strip() + if not base: + raise SystemExit( + "error: PLANE_EE_API_DIR is required (path to plane-ee apps/api, or monorepo root with apps/api/...)" + ) + root = Path(base).expanduser().resolve() + candidates = [ + root / "plane" / "payment" / "flags" / "flag.py", + root / "apps" / "api" / "plane" / "payment" / "flags" / "flag.py", + ] + for c in candidates: + if c.is_file(): + return c + raise SystemExit( + f"error: FeatureFlag module not found under PLANE_EE_API_DIR={root}; " + f"tried: {', '.join(str(c) for c in candidates)}" + ) + + +def load_feature_flag_values() -> dict[str, bool]: + """Load FeatureFlag enum values and return {value: True} for every flag.""" + flag_path = _resolve_flag_module_path() + # Put the API package root on sys.path so relative imports inside flag.py work + # when the module itself only needs the enum. + api_root = flag_path.parents[3] # .../plane + package_root = flag_path.parents[4] # .../apps/api or similar + for p in (str(package_root), str(api_root.parent)): + if p not in sys.path: + sys.path.insert(0, p) + + spec = importlib.util.spec_from_file_location("plane_eval_flag", flag_path) + if spec is None or spec.loader is None: + raise SystemExit(f"error: cannot load flag module from {flag_path}") + flag_mod = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(flag_mod) + except Exception as exc: + # Fallback: read the file and eval only the enum body if full import fails + # (Django settings). Try a minimal AST-free scrape of string values. + text = flag_path.read_text(encoding="utf-8") + values: dict[str, bool] = {} + for line in text.splitlines(): + line = line.strip() + # e.g. FOO = "FOO" or FOO = "some-flag" + if "=" in line and not line.startswith("#") and not line.startswith("class"): + _, _, rhs = line.partition("=") + rhs = rhs.strip().rstrip(",") + if len(rhs) >= 2 and rhs[0] in "\"'" and rhs[-1] == rhs[0]: + values[rhs[1:-1]] = True + if values: + print( + f"mock flag server: loaded {len(values)} flags via scrape (import failed: {exc})", + flush=True, + ) + return values + raise SystemExit(f"error: failed to import FeatureFlag from {flag_path}: {exc}") from exc + + FeatureFlag = getattr(flag_mod, "FeatureFlag", None) + if FeatureFlag is None: + raise SystemExit(f"error: FeatureFlag not found in {flag_path}") + return {f.value: True for f in FeatureFlag} + + +def main(argv: list[str] | None = None) -> int: + host = "127.0.0.1" + port = 9911 + if argv is None: + argv = sys.argv[1:] + if len(argv) >= 1: + port = int(argv[0]) + + values = load_feature_flag_values() + print(f"mock flag server: {len(values)} flags all-on on {host}:{port}", flush=True) + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + body = json.dumps({"values": values}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + body = json.dumps({"values": values, "ok": True}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + # Threaded: Django dev server is multi-threaded and fans out flag lookups. + ThreadingHTTPServer((host, port), Handler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/proxy.py b/evals/proxy.py new file mode 100644 index 00000000..57f6ae2c --- /dev/null +++ b/evals/proxy.py @@ -0,0 +1,587 @@ +"""Stdio MCP recording proxy — byte-faithful JSON-RPC relay with sidecar call log. + +Usage: + python -m evals.proxy --log SIDECAR.jsonl -- + +Spawns the target as a child, relays parent stdin → child stdin and child +stdout → parent stdout as raw bytes (byte-faithful; does not re-serialize). +Parses complete newline-delimited JSON lines from a *copy* of each direction +to record ``tools/call`` request/response pairs into the sidecar JSONL. + +I/O uses ``os.read`` on raw fds + per-direction bytearray buffers — never +``select`` + buffered ``readline`` (partial lines would hang; buffered +prefetch would stall multi-line clients). + +Child stderr is forwarded to our stderr. Exit code matches the child +(negative/signal codes map to conventional 128+signum). +""" + +from __future__ import annotations + +import argparse +import json +import os +import select +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any + +# Single post-EOF / child-exit deadline for the whole shutdown sequence. +SHUTDOWN_DEADLINE_S = 10.0 +READ_CHUNK = 65536 + +# Repo root for PYTHONPATH scrubbing (parent of evals/). +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Stdio MCP recording proxy (tools/call → sidecar JSONL)", + ) + p.add_argument( + "--log", + required=True, + type=Path, + help="Sidecar JSONL path for recorded tool calls + proxy_meta summary", + ) + p.add_argument( + "command", + nargs=argparse.REMAINDER, + help="Target MCP server command after --", + ) + args = p.parse_args(argv) + cmd = list(args.command or []) + if cmd and cmd[0] == "--": + cmd = cmd[1:] + if not cmd: + p.error("target command required after --") + args.command = cmd + return args + + +def map_child_returncode(rc: int | None) -> int: + """Map subprocess returncode to a conventional shell exit status. + + Negative codes mean killed by signal N (``-N``); return ``128 + N``. + """ + if rc is None: + return 1 + if rc < 0: + return 128 + (-rc) + return int(rc) + + +def write_all_fd(fd: int, data: bytes) -> None: + """Write ``data`` fully to a raw fd, looping on short writes.""" + view = memoryview(data) + offset = 0 + while offset < len(view): + n = os.write(fd, view[offset:]) + if n == 0: + raise BrokenPipeError("os.write returned 0") + offset += n + + +def scrub_child_pythonpath(env: dict[str, str] | None = None) -> dict[str, str]: + """Return a copy of ``env`` with this repo's root removed from PYTHONPATH. + + The proxy may be launched via ``python -m evals.proxy`` with PYTHONPATH set + to the monorepo root so ``evals`` is importable from a foreign cwd. That + entry must not leak into the *real* MCP server child (which may resolve + ``plane_mcp`` from its own venv). + """ + base = dict(env if env is not None else os.environ) + root = str(REPO_ROOT) + raw = base.get("PYTHONPATH", "") + if not raw: + return base + parts = [p for p in raw.split(os.pathsep) if p and Path(p).resolve() != REPO_ROOT.resolve()] + # Also drop exact string matches that may not resolve the same way. + parts = [p for p in parts if p != root] + if parts: + base["PYTHONPATH"] = os.pathsep.join(parts) + else: + base.pop("PYTHONPATH", None) + return base + + +class SidecarRecorder: + """Thread-safe recorder for tools/call pairs into a JSONL sidecar. + + Finalization is atomic under ``_lock``: once ``write_meta`` sets + ``finalized``, further row appends no-op so ``proxy_meta`` is always the + last sidecar line even if daemon pumps keep running briefly. + """ + + def __init__(self, log_path: Path) -> None: + self.log_path = log_path + self._lock = threading.Lock() + self._pending: dict[Any, dict[str, Any]] = {} + self._seq = 0 + self.relayed_lines = 0 + self.unparsed_lines = 0 + self.unmatched_responses = 0 + self.notifications = 0 + self.server_requests = 0 + self.child_killed = False + self.pumps_alive = False + self.finalized = False + # Post-finalize append attempts (not written; for tests / diagnostics). + self.post_finalize_appends = 0 + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def _append(self, row: dict[str, Any]) -> None: + line = json.dumps(row, default=str, ensure_ascii=False) + "\n" + with self._lock: + if self.finalized: + self.post_finalize_appends += 1 + return + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(line) + + def note_relayed(self) -> None: + with self._lock: + self.relayed_lines += 1 + + def note_unparsed(self) -> None: + with self._lock: + self.unparsed_lines += 1 + self.relayed_lines += 1 + + def on_client_message(self, obj: dict[str, Any]) -> None: + """Handle a parsed JSON-RPC message from the client (parent → child).""" + has_method = "method" in obj + has_id = "id" in obj + if has_method and not has_id: + with self._lock: + self.notifications += 1 + return + if not has_method: + return + method = obj.get("method") + if method != "tools/call": + return + params = obj.get("params") or {} + if not isinstance(params, dict): + params = {} + name = params.get("name") or "" + arguments = params.get("arguments") + if arguments is None: + arguments = {} + req_id = obj.get("id") + with self._lock: + self._seq += 1 + self._pending[req_id] = { + "tool": str(name), + "args": arguments, + "t_start": time.perf_counter(), + "seq": self._seq, + } + + def on_server_message(self, obj: dict[str, Any]) -> None: + """Handle a parsed JSON-RPC message from the server (child → parent).""" + has_method = "method" in obj + has_id = "id" in obj + + if has_method and not has_id: + with self._lock: + self.notifications += 1 + return + if has_method and has_id: + with self._lock: + self.server_requests += 1 + return + if not has_id: + return + + req_id = obj.get("id") + with self._lock: + pending = self._pending.pop(req_id, None) + if pending is None: + with self._lock: + self.unmatched_responses += 1 + return + duration_ms = int(round((time.perf_counter() - pending["t_start"]) * 1000)) + if "error" in obj: + is_error = True + result_payload = obj.get("error") + else: + result = obj.get("result") + is_error = False + if isinstance(result, dict): + is_error = bool(result.get("isError") or result.get("is_error")) + result_payload = result + try: + result_chars = len(json.dumps(result_payload, default=str, ensure_ascii=False)) + except Exception: + result_chars = len(str(result_payload)) + self._append( + { + "tool": pending["tool"], + "args": pending["args"], + "is_error": is_error, + "result_chars": result_chars, + "duration_ms": duration_ms, + "seq": pending["seq"], + } + ) + + def write_meta(self) -> None: + """Write proxy_meta as the last row and seal the sidecar (atomic under lock).""" + with self._lock: + if self.finalized: + self.post_finalize_appends += 1 + return + row = { + "row_type": "proxy_meta", + "relayed_lines": self.relayed_lines, + "unparsed_lines": self.unparsed_lines, + "unmatched_responses": self.unmatched_responses, + "notifications": self.notifications, + "server_requests": self.server_requests, + "pending_left": len(self._pending), + "child_killed": self.child_killed, + "pumps_alive": self.pumps_alive, + } + line = json.dumps(row, default=str, ensure_ascii=False) + "\n" + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(line) + self.finalized = True + + +def try_parse_json_line(line: bytes) -> dict[str, Any] | None: + """Parse a JSON object line; return None on failure (never raises).""" + try: + text = line.decode("utf-8").strip() + except UnicodeDecodeError: + return None + if not text or not text.startswith("{"): + return None + try: + obj = json.loads(text) + except json.JSONDecodeError: + return None + return obj if isinstance(obj, dict) else None + + +def process_buffer_lines( + buf: bytearray, + *, + forward_fd: int, + recorder: SidecarRecorder | None, + is_client: bool, + record_jsonrpc: bool, +) -> None: + """Split complete lines from ``buf``, record then forward, leave incomplete tail. + + **Record-before-forward**: for JSON-RPC directions, update the sidecar / + pending map *before* the line becomes visible to the opposite endpoint. + A fast child responding on stdout must never race past an unregistered + pending tools/call id; a failed parent write must not lose a completed + response that was already matched. + """ + while True: + idx = buf.find(b"\n") + if idx < 0: + break + line = bytes(buf[: idx + 1]) + del buf[: idx + 1] + if record_jsonrpc and recorder is not None: + obj = try_parse_json_line(line) + if obj is None: + recorder.note_unparsed() + else: + recorder.note_relayed() + try: + if is_client: + recorder.on_client_message(obj) + else: + recorder.on_server_message(obj) + except Exception: + pass + # Forward only after recording so the opposite endpoint cannot race. + write_all_fd(forward_fd, line) + + +def pump_raw( + *, + read_fd: int, + write_fd: int, + recorder: SidecarRecorder | None, + is_client: bool, + record_jsonrpc: bool, + cancel: threading.Event | None, + done: threading.Event, +) -> None: + """Byte-faithful pump: ``os.read`` + line buffer; optional cancel for stdin only. + + Stdout/stderr pumps pass ``cancel=None`` and drain until ``os.read`` returns + ``b""`` (pipe EOF) so final responses after child exit are not dropped. + Never uses buffered TextIO wrappers with select. + """ + buf = bytearray() + try: + while True: + if cancel is not None and cancel.is_set(): + break + try: + ready, _, _ = select.select([read_fd], [], [], 0.2) + except (ValueError, OSError): + break + if not ready: + continue + try: + chunk = os.read(read_fd, READ_CHUNK) + except OSError: + break + if not chunk: + break + buf.extend(chunk) + try: + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=is_client, + record_jsonrpc=record_jsonrpc, + ) + except (BrokenPipeError, OSError): + break + # Flush remaining complete lines, then any partial tail (byte-faithful). + try: + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=is_client, + record_jsonrpc=record_jsonrpc, + ) + except (BrokenPipeError, OSError): + pass + if buf: + try: + write_all_fd(write_fd, bytes(buf)) + except (BrokenPipeError, OSError): + pass + if record_jsonrpc and recorder is not None: + recorder.note_unparsed() + buf.clear() + finally: + done.set() + + +def _remaining(deadline_at: float) -> float: + """Seconds left until ``deadline_at`` (never negative).""" + return max(0.0, deadline_at - time.monotonic()) + + +def reap_timeout(deadline_at: float | None, floor: float = 0.1) -> float: + """Timeout for kill/reap waits: remaining budget, never below ``floor``. + + When the overall deadline is exhausted, still allow a short reap so kill + is not skipped entirely. + """ + if deadline_at is None: + return floor + return max(floor, _remaining(deadline_at)) + + +def run_proxy(command: list[str], log_path: Path) -> int: + """Spawn ``command`` as the real MCP server and relay with recording. + + Returns the child's exit code (or 1 on spawn failure). Guarantees + ``proxy_meta`` is the last sidecar row and the child is reaped even on + crash paths. Pump threads are daemon so a blocked write cannot hold the + process past the shutdown deadline. + """ + recorder = SidecarRecorder(log_path) + child: subprocess.Popen[bytes] | None = None + # Scrub repo PYTHONPATH so the real server does not import from this tree. + child_env = scrub_child_pythonpath() + t_in = t_out = t_err = None + stdin_done = stdout_done = stderr_done = None + deadline_at: float | None = None + try: + try: + child = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + env=child_env, + ) + except OSError as exc: + print(f"evals.proxy: failed to spawn {command!r}: {exc}", file=sys.stderr) + return 1 + + assert child.stdin is not None and child.stdout is not None and child.stderr is not None + # Raw fds — never use the buffered TextIO wrappers with select. + child_stdin_fd = child.stdin.fileno() + child_stdout_fd = child.stdout.fileno() + child_stderr_fd = child.stderr.fileno() + parent_stdin_fd = sys.stdin.fileno() + parent_stdout_fd = sys.stdout.fileno() + parent_stderr_fd = sys.stderr.fileno() + + cancel_stdin = threading.Event() + stdin_done = threading.Event() + stdout_done = threading.Event() + stderr_done = threading.Event() + + # daemon=True: a pump blocked writing to an undrained parent cannot + # keep the process alive past the shutdown deadline. + t_in = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": parent_stdin_fd, + "write_fd": child_stdin_fd, + "recorder": recorder, + "is_client": True, + "record_jsonrpc": True, + "cancel": cancel_stdin, + "done": stdin_done, + }, + name="proxy-stdin", + daemon=True, + ) + t_out = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": child_stdout_fd, + "write_fd": parent_stdout_fd, + "recorder": recorder, + "is_client": False, + "record_jsonrpc": True, + "cancel": None, # drain until pipe EOF — never gate on cancel + "done": stdout_done, + }, + name="proxy-stdout", + daemon=True, + ) + t_err = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": child_stderr_fd, + "write_fd": parent_stderr_fd, + "recorder": None, + "is_client": False, + "record_jsonrpc": False, # stderr is not JSON-RPC + "cancel": None, + "done": stderr_done, + }, + name="proxy-stderr", + daemon=True, + ) + t_in.start() + t_out.start() + t_err.start() + + # Phase 1: run until client stdin EOF or child exits. + # Child exit cancels the *stdin* pump only — stdout must drain to pipe EOF. + while child.poll() is None and not stdin_done.is_set(): + time.sleep(0.05) + + # One deadline for the entire post-EOF / post-child-exit shutdown. + deadline_at = time.monotonic() + SHUTDOWN_DEADLINE_S + cancel_stdin.set() + try: + os.close(child_stdin_fd) + except OSError: + pass + + # Phase 2: wait for pumps (remaining time only — no stacked fixed timeouts). + while _remaining(deadline_at) > 0: + if stdout_done.is_set() and stderr_done.is_set() and stdin_done.is_set(): + break + if child.poll() is not None and stdout_done.is_set() and stderr_done.is_set(): + break + time.sleep(min(0.05, max(0.01, _remaining(deadline_at)))) + + rem = _remaining(deadline_at) + if rem > 0: + t_in.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0: + t_out.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0: + t_err.join(timeout=rem) + + if child.poll() is None: + rem = _remaining(deadline_at) + if rem > 0: + try: + child.wait(timeout=rem) + except subprocess.TimeoutExpired: + recorder.child_killed = True + child.kill() + # Bounded wait after kill — remaining budget with floor. + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + else: + recorder.child_killed = True + child.kill() + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + + # After kill/exit, join stdout/stderr again (bounded) so meta is last. + rem = _remaining(deadline_at) + if rem > 0 and t_out is not None: + t_out.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0 and t_err is not None: + t_err.join(timeout=rem) + + pumps_still = any(t is not None and t.is_alive() for t in (t_in, t_out, t_err)) or not all( + e.is_set() if e is not None else True for e in (stdin_done, stdout_done, stderr_done) + ) + recorder.pumps_alive = pumps_still + return map_child_returncode(child.returncode) + finally: + if child is not None and child.poll() is None: + try: + recorder.child_killed = True + child.kill() + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + except Exception: + pass + # If pumps are still alive at deadline, note it; meta is still last row + # (finalized flag drops any further appends from daemon pumps). + if t_out is not None or t_err is not None or t_in is not None: + still = any(t is not None and t.is_alive() for t in (t_in, t_out, t_err)) + if still: + recorder.pumps_alive = True + try: + recorder.write_meta() + except Exception as exc: + print(f"evals.proxy: failed to write proxy_meta: {exc}", file=sys.stderr) + + +def main(argv: list[str] | None = None) -> int: + # Detach from the CLI's process group so a harness timeout killpg on the + # agent CLI does not SIGKILL this proxy (+ MCP child). After setsid we are + # our own session/group leader; CLI group kill leaves us alive to see stdin + # EOF, flush rows, and write proxy_meta within the shutdown deadline. + try: + os.setsid() + except OSError: + # Already a session leader, or platform forbids setsid — continue. + pass + args = parse_args(argv) + return run_proxy(list(args.command), Path(args.log)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/report.py b/evals/report.py new file mode 100644 index 00000000..cc3843d7 --- /dev/null +++ b/evals/report.py @@ -0,0 +1,645 @@ +"""Summary table, A/B delta, and multi-surface tables for eval JSONL results. + +Usage: + python -m evals.report evals/results/A.jsonl + python -m evals.report A.jsonl B.jsonl # A/B delta (sign test + Wilson) + python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface + python -m evals.report --table --markdown f1.jsonl f2.jsonl +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Literal + +from evals.tasks import TASKS_BY_ID + +DedupeMode = Literal["latest", "none"] + + +def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """95% Wilson score interval for a binomial proportion.""" + if n <= 0: + return (0.0, 0.0) + p = k / n + z2 = z * z + denom = 1.0 + z2 / n + centre = p + z2 / (2.0 * n) + margin = z * math.sqrt((p * (1.0 - p) + z2 / (4.0 * n)) / n) + lo = max(0.0, (centre - margin) / denom) + hi = min(1.0, (centre + margin) / denom) + return (lo, hi) + + +def sign_test_pvalue(deltas: list[float]) -> float | None: + """Two-sided exact binomial sign test on non-zero paired deltas. + + H0: P(delta > 0) = 1/2. Zero deltas are dropped. Returns None when no + non-zero pairs remain. Uses ``math.comb`` only (no scipy). + """ + nonzero = [d for d in deltas if d != 0] + n = len(nonzero) + if n == 0: + return None + k = sum(1 for d in nonzero if d > 0) + total = 2**n + # Two-sided: 2 * min(left cdf, right survival), capped at 1. + left = sum(math.comb(n, i) for i in range(0, k + 1)) / total + right = sum(math.comb(n, i) for i in range(k, n + 1)) / total + return min(1.0, 2.0 * min(left, right)) + + +def _median(xs: list[float]) -> float | None: + if not xs: + return None + s = sorted(xs) + m = len(s) // 2 + if len(s) % 2: + return float(s[m]) + return (s[m - 1] + s[m]) / 2.0 + + +def _percentile(xs: list[float], p: float) -> float | None: + if not xs: + return None + s = sorted(xs) + if len(s) == 1: + return float(s[0]) + k = (len(s) - 1) * p + f = math.floor(k) + c = math.ceil(k) + if f == c: + return float(s[int(k)]) + return float(s[f] + (s[c] - s[f]) * (k - f)) + + +def _iqr(xs: list[float]) -> tuple[float | None, float | None, float | None]: + return (_percentile(xs, 0.25), _median(xs), _percentile(xs, 0.75)) + + +def is_meta_row(row: dict[str, Any]) -> bool: + """True for run-header meta lines (or any row without a task_id).""" + if row.get("row_type") == "meta": + return True + return row.get("task_id") is None + + +def is_infra_error_row(row: dict[str, Any]) -> bool: + """True when a row failed for infrastructure reasons (seed/cli/sdk), not task verify. + + Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, + ``infra_sdk``, …) is excluded from success-rate denominators. + """ + ec = row.get("error_class") + return isinstance(ec, str) and ec.startswith("infra_") + + +def dedupe_rows_latest(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Keep only the last row per (task_id, rep, surface); preserve key insertion order.""" + latest: dict[tuple[Any, Any, Any], dict[str, Any]] = {} + order: list[tuple[Any, Any, Any]] = [] + for r in rows: + key = (r.get("task_id"), r.get("rep"), r.get("surface")) + if key not in latest: + order.append(key) + latest[key] = r + return [latest[k] for k in order] + + +def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[dict[str, Any]]: + """Load JSONL data rows (skip meta / missing task_id). + + Default ``dedupe="latest"`` keeps the last row per (task_id, rep, surface) + so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. + """ + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as fh: + for line_no, line in enumerate(fh, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: {path}:{line_no}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict) or is_meta_row(row): + continue + rows.append(row) + if dedupe == "latest": + return dedupe_rows_latest(rows) + # Forensics: warn on duplicates but keep all. + seen_keys: set[tuple[Any, Any, Any]] = set() + for r in rows: + key = (r.get("task_id"), r.get("rep"), r.get("surface")) + if key in seen_keys: + print( + f"warning: {path}: duplicate (task_id, rep, surface)={key} " + f"(--no-dedupe keeps all rows; bare --out reuse double-counts)", + file=sys.stderr, + ) + else: + seen_keys.add(key) + return rows + + +def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Aggregate per-task metrics. + + Rows with ``error_class`` starting ``infra_`` are excluded from success-rate + denominators and counted separately as ``infra_errors`` (total on the returned + dict under the special key ``_meta``). Other non-null ``error`` rows remain + harness errors (excluded from success, counted in ``harness_err``). + """ + by_task: dict[str, list[dict[str, Any]]] = defaultdict(list) + harness_err_by_task: dict[str, int] = defaultdict(int) + infra_err_by_task: dict[str, int] = defaultdict(int) + infra_errors = 0 + for r in rows: + if is_meta_row(r): + continue + tid = r["task_id"] + if is_infra_error_row(r): + infra_errors += 1 + infra_err_by_task[tid] += 1 + continue # infra seed/cli — excluded from success aggregates + if r.get("error"): + harness_err_by_task[tid] += 1 + continue # harness/API errors excluded from success/medians (F4) + if r.get("skipped"): + continue # skipped rows are excluded from success denominators + by_task[tid].append(r) + + # Include tasks that only had harness/infra errors so columns stay visible. + all_task_ids = sorted(set(by_task) | set(harness_err_by_task) | set(infra_err_by_task)) + + out: dict[str, dict[str, Any]] = {} + total_k = 0 + total_n = 0 + for task_id in all_task_ids: + trs = by_task.get(task_id, []) + n = len(trs) + k = sum(1 for r in trs if r.get("success")) + total_k += k + total_n += n + lo, hi = wilson_interval(k, n) if n else (0.0, 0.0) + calls = [float(r.get("num_calls") or 0) for r in trs] + q1, med_calls, q3 = _iqr(calls) + min_calls = min(calls) if calls else None + max_calls = max(calls) if calls else None + optimal = TASKS_BY_ID.get(task_id, {}).get("optimal_calls") + total_calls = 0 + mispick = 0 + errored = 0 + result_tokens: list[float] = [] + for r in trs: + for c in r.get("calls") or []: + total_calls += 1 + if c.get("class") in ("alternate", "out_of_set"): + mispick += 1 + if c.get("is_error"): + errored += 1 + if c.get("result_tokens") is not None: + result_tokens.append(float(c["result_tokens"])) + capped = sum(1 for r in trs if r.get("hit_max_iterations") or r.get("stop_reason") == "max_tokens") + cum_inputs = [float(r.get("cum_input_tokens") or 0) for r in trs] + out[task_id] = { + "n": n, + "k": k, + "success": f"{k}/{n}" if n else "0/0", + "wilson_lo": lo, + "wilson_hi": hi, + "med_calls": med_calls, + "calls_min": min_calls, + "calls_max": max_calls, + "calls_q1": q1, + "calls_q3": q3, + "optimal_calls": optimal, + "mispick_rate": (mispick / total_calls) if total_calls else 0.0, + "errored_calls": errored, + "capped": capped, + "harness_err": harness_err_by_task.get(task_id, 0), + "infra_err": infra_err_by_task.get(task_id, 0), + "med_result_tokens": _median(result_tokens), + "p95_result_tokens": _percentile(result_tokens, 0.95), + "med_cum_input": _median(cum_inputs), + } + agg_lo, agg_hi = wilson_interval(total_k, total_n) if total_n else (0.0, 0.0) + out["_meta"] = { + "infra_errors": infra_errors, + "aggregate_k": total_k, + "aggregate_n": total_n, + "aggregate_wilson_lo": agg_lo, + "aggregate_wilson_hi": agg_hi, + } + return out + + +def _fmt(x: float | None, digits: int = 1) -> str: + if x is None: + return "-" + return f"{x:.{digits}f}" + + +def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: + meta = summary.get("_meta") or {} + print(title) + if meta.get("infra_errors"): + print(f"infra errors: {meta['infra_errors']}") + agg_n = int(meta.get("aggregate_n") or 0) + if agg_n: + agg_k = int(meta.get("aggregate_k") or 0) + alo = float(meta.get("aggregate_wilson_lo") or 0.0) + ahi = float(meta.get("aggregate_wilson_hi") or 0.0) + rate = agg_k / agg_n if agg_n else 0.0 + print(f"aggregate success: {agg_k}/{agg_n} ({rate:.1%}) Wilson95 [{alo:.2f},{ahi:.2f}]") + # Show min/med/max call columns when any task has n>1. + show_var = any(s.get("n", 0) > 1 for tid, s in summary.items() if tid != "_meta") + if show_var: + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " + f"{'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} {'med_rtok':>8} {'p95_rtok':>8} {'med_cum_in':>10}" + ) + else: + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{'med_calls':>9} {'opt':>4} {'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} {'med_rtok':>8} {'p95_rtok':>8} {'med_cum_in':>10}" + ) + print(header) + print("-" * len(header)) + for task_id, s in summary.items(): + if task_id == "_meta": + continue + wilson = f"[{s['wilson_lo']:.2f},{s['wilson_hi']:.2f}]" + iqr = f"{_fmt(s['calls_q1'])}-{_fmt(s['calls_q3'])}" + opt = s["optimal_calls"] if s["optimal_calls"] is not None else "-" + if show_var: + print( + f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " + f"{_fmt(s.get('calls_min')):>9} {_fmt(s['med_calls']):>9} {_fmt(s.get('calls_max')):>9} " + f"{opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " + f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " + f"{s.get('infra_err', 0):>5} " + f"{_fmt(s['med_result_tokens'], 0):>8} {_fmt(s['p95_result_tokens'], 0):>8} " + f"{_fmt(s['med_cum_input'], 0):>10}" + ) + else: + print( + f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " + f"{_fmt(s['med_calls']):>9} {opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " + f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " + f"{s.get('infra_err', 0):>5} " + f"{_fmt(s['med_result_tokens'], 0):>8} {_fmt(s['p95_result_tokens'], 0):>8} " + f"{_fmt(s['med_cum_input'], 0):>10}" + ) + + +# --------------------------------------------------------------------------- +# A/B comparison +# --------------------------------------------------------------------------- + + +def ab_compare( + rows_a: list[dict[str, Any]], + rows_b: list[dict[str, Any]], +) -> dict[str, Any]: + """Compare two result sets: paired call-count deltas + success rates. + + Paired call deltas only include tasks that are present and successful in + both A and B. When multiple success rows exist for a task, the **last** one + wins (matches load-time ``dedupe="latest"`` semantics). + """ + sum_a = summarize(rows_a) + sum_b = summarize(rows_b) + + def _success_rows_by_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for r in rows: + if is_meta_row(r) or is_infra_error_row(r) or r.get("error") or r.get("skipped"): + continue + if not r.get("success"): + continue + tid = str(r["task_id"]) + out[tid] = r # last wins (dedupe already applied) + return out + + sa = _success_rows_by_task(rows_a) + sb = _success_rows_by_task(rows_b) + shared = sorted(set(sa) & set(sb)) + deltas: list[float] = [] + per_task: list[dict[str, Any]] = [] + for tid in shared: + ca = float(sa[tid].get("num_calls") or 0) + cb = float(sb[tid].get("num_calls") or 0) + d = cb - ca # B − A (negative = B fewer calls = better if lower is better) + deltas.append(d) + per_task.append({"task_id": tid, "calls_a": ca, "calls_b": cb, "delta": d}) + + meta_a = sum_a.get("_meta") or {} + meta_b = sum_b.get("_meta") or {} + return { + "summary_a": sum_a, + "summary_b": sum_b, + "paired_tasks": per_task, + "median_delta": _median(deltas), + "sign_test_p": sign_test_pvalue(deltas), + "n_paired": len(deltas), + "success_a": { + "k": int(meta_a.get("aggregate_k") or 0), + "n": int(meta_a.get("aggregate_n") or 0), + "wilson": ( + float(meta_a.get("aggregate_wilson_lo") or 0.0), + float(meta_a.get("aggregate_wilson_hi") or 0.0), + ), + }, + "success_b": { + "k": int(meta_b.get("aggregate_k") or 0), + "n": int(meta_b.get("aggregate_n") or 0), + "wilson": ( + float(meta_b.get("aggregate_wilson_lo") or 0.0), + float(meta_b.get("aggregate_wilson_hi") or 0.0), + ), + }, + } + + +def print_ab_report(cmp: dict[str, Any], path_a: Path, path_b: Path) -> None: + print(f"A/B compare: A={path_a} B={path_b}") + sa, sb = cmp["success_a"], cmp["success_b"] + ra = (sa["k"] / sa["n"]) if sa["n"] else 0.0 + rb = (sb["k"] / sb["n"]) if sb["n"] else 0.0 + print(f" success A: {sa['k']}/{sa['n']} ({ra:.1%}) Wilson95 [{sa['wilson'][0]:.2f},{sa['wilson'][1]:.2f}]") + print(f" success B: {sb['k']}/{sb['n']} ({rb:.1%}) Wilson95 [{sb['wilson'][0]:.2f},{sb['wilson'][1]:.2f}]") + print(f" success rate delta (B−A): {rb - ra:+.1%}") + print(f" paired successful tasks: {cmp['n_paired']}") + print(f" median call delta (B−A): {_fmt(cmp['median_delta'])}") + p = cmp["sign_test_p"] + print(f" sign-test p-value (two-sided): {p if p is not None else 'n/a'}") + if cmp["paired_tasks"]: + print() + print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") + print("-" * 34) + for row in cmp["paired_tasks"]: + print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") + + +# --------------------------------------------------------------------------- +# Multi-surface table +# --------------------------------------------------------------------------- + + +def _task_sort_key(tid: str) -> tuple[str, int]: + digits = "".join(c for c in tid if c.isdigit()) + return (tid[0] if tid else "", int(digits) if digits else 0) + + +def format_surface_cell(row: dict[str, Any] | None) -> str: + """Cell for multi-surface table: '✅ Nc/Mmp', 'skip', 'ERR', or '—'.""" + if row is None: + return "—" + if row.get("skipped"): + return "skip" + if row.get("error") or is_infra_error_row(row): + return "ERR" + ok = "✅" if row.get("success") else "❌" + n_calls = row.get("num_calls") + n_calls_s = str(n_calls) if n_calls is not None else "?" + if row.get("classification") == "external": + return f"{ok} {n_calls_s}c" + alt = row.get("alternate_calls") + oos = row.get("out_of_set_calls") + # None counters (external nulling) → omit mispick suffix. + if alt is None and oos is None: + return f"{ok} {n_calls_s}c" + mp = int(alt or 0) + int(oos or 0) + if mp: + return f"{ok} {n_calls_s}c/{mp}mp" + return f"{ok} {n_calls_s}c" + + +def build_multi_surface_table( + file_rows: list[tuple[str, list[dict[str, Any]]]], +) -> dict[str, Any]: + """Build a per-task × per-surface grid from labeled row sets. + + ``file_rows`` is a list of ``(column_label, rows)``. Column labels default + to each file's dominant ``surface`` field when the caller passes that label. + For each column, the latest row per task_id is used (rep-agnostic: last wins). + """ + columns: list[str] = [] + by_col: dict[str, dict[str, dict[str, Any]]] = {} + for label, rows in file_rows: + columns.append(label) + col_map: dict[str, dict[str, Any]] = {} + for r in rows: + if is_meta_row(r): + continue + tid = str(r["task_id"]) + col_map[tid] = r # last wins + by_col[label] = col_map + + all_tasks = sorted({t for m in by_col.values() for t in m}, key=_task_sort_key) + cells: dict[str, dict[str, str]] = {} + raw: dict[str, dict[str, dict[str, Any] | None]] = {} + for tid in all_tasks: + cells[tid] = {} + raw[tid] = {} + for col in columns: + r = by_col[col].get(tid) + raw[tid][col] = r + cells[tid][col] = format_surface_cell(r) + + # Aggregate footer per column. + footer: dict[str, dict[str, Any]] = {} + for col in columns: + succ = run = calls = mispicks = 0 + mispick_comparable = True + infra = 0 + for _tid, r in by_col[col].items(): + if is_infra_error_row(r): + infra += 1 + continue + if r.get("error"): + continue + if r.get("skipped"): + continue + run += 1 + if r.get("success"): + succ += 1 + calls += int(r.get("num_calls") or 0) + if r.get("classification") == "external": + mispick_comparable = False + else: + alt, oos = r.get("alternate_calls"), r.get("out_of_set_calls") + if alt is None and oos is None: + mispick_comparable = False + else: + mispicks += int(alt or 0) + int(oos or 0) + footer[col] = { + "success": succ, + "n": run, + "calls": calls, + "mispicks": mispicks if mispick_comparable else None, + "infra_errors": infra, + } + return {"columns": columns, "task_ids": all_tasks, "cells": cells, "raw": raw, "footer": footer} + + +def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) -> str: + """Render multi-surface table as plain text or GitHub markdown.""" + cols: list[str] = table["columns"] + task_ids: list[str] = table["task_ids"] + cells: dict[str, dict[str, str]] = table["cells"] + footer: dict[str, dict[str, Any]] = table["footer"] + lines: list[str] = [] + + def _prompt_snip(tid: str) -> str: + p = (TASKS_BY_ID.get(tid, {}).get("prompt") or "").replace("{project}", "P") + return (p[:32] + "…") if len(p) > 32 else p + + if markdown: + header = "| task | what | " + " | ".join(cols) + " |" + sep = "| --- | --- | " + " | ".join("---" for _ in cols) + " |" + lines.append(header) + lines.append(sep) + for tid in task_ids: + row_cells = " | ".join(cells[tid].get(c, "—") for c in cols) + lines.append(f"| {tid} | {_prompt_snip(tid)} | {row_cells} |") + # Footer + foot_parts = [] + for c in cols: + f = footer[c] + rate = f"{f['success']}/{f['n']}" if f["n"] else "0/0" + mp = f", {f['mispicks']}mp" if f["mispicks"] is not None else "" + foot_parts.append(f"{rate} ({f['calls']}c{mp}, i={f['infra_errors']})") + lines.append("| **agg** | | " + " | ".join(foot_parts) + " |") + return "\n".join(lines) + "\n" + + col_w = max(14, max((len(c) for c in cols), default=14)) + head = f"{'task':5} {'what':34} " + " ".join(f"{c:{col_w}}" for c in cols) + lines.append(head) + lines.append("-" * len(head)) + for tid in task_ids: + line = f"{tid:5} {_prompt_snip(tid):34} " + for c in cols: + line += f"{cells[tid].get(c, '—'):{col_w}} " + lines.append(line.rstrip()) + lines.append("-" * len(head)) + for c in cols: + f = footer[c] + rate = f"{f['success']}/{f['n']}" if f["n"] else "0/0" + pct = f" ({100 * f['success'] / f['n']:.0f}%)" if f["n"] else "" + mp = f" mispicks {f['mispicks']}" if f["mispicks"] is not None else " mispicks n/a" + lines.append(f"{c:12} success {rate}{pct} total calls {f['calls']}{mp} infra {f['infra_errors']}") + return "\n".join(lines) + "\n" + + +def _surface_label_for_file(path: Path, rows: list[dict[str, Any]]) -> str: + """Pick a column label from the file's dominant surface field, else stem.""" + counts: dict[str, int] = defaultdict(int) + for r in rows: + s = r.get("surface") + if s: + counts[str(s)] += 1 + if counts: + return max(counts, key=counts.get) # type: ignore[arg-type] + return path.stem + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description="Summarize eval JSONL results") + p.add_argument( + "files", + nargs="*", + help="JSONL file(s): one for summary, two for A/B, N with --table", + ) + p.add_argument( + "--table", + action="store_true", + help="Multi-surface per-task table (one column per file, labeled by surface)", + ) + p.add_argument( + "--markdown", + action="store_true", + help="With --table, emit a GitHub-flavored markdown table", + ) + p.add_argument( + "--no-dedupe", + action="store_true", + help="Keep all rows (forensics); default is latest-wins per (task_id,rep,surface)", + ) + args = p.parse_args(argv) + dedupe: DedupeMode = "none" if args.no_dedupe else "latest" + + if not args.files: + p.print_help() + return 2 + + paths = [Path(f) for f in args.files] + for path in paths: + if not path.exists(): + print(f"error: file not found: {path}", file=sys.stderr) + return 2 + + if args.table: + if len(paths) < 1: + print("error: --table requires at least one JSONL", file=sys.stderr) + return 2 + labeled: list[tuple[str, list[dict[str, Any]]]] = [] + used_labels: set[str] = set() + for path in paths: + rows = load_rows(path, dedupe=dedupe) + label = _surface_label_for_file(path, rows) + # Disambiguate duplicate surface labels (e.g. two external files). + base = label + n = 2 + while label in used_labels: + label = f"{base}-{n}" + n += 1 + used_labels.add(label) + labeled.append((label, rows)) + table = build_multi_surface_table(labeled) + sys.stdout.write(render_multi_surface_table(table, markdown=args.markdown)) + return 0 + + if len(paths) == 1: + path = paths[0] + rows = load_rows(path, dedupe=dedupe) + summary = summarize(rows) + task_keys = [k for k in summary if k != "_meta"] + if not task_keys: + infra_n = (summary.get("_meta") or {}).get("infra_errors", 0) + if infra_n: + print(f"infra errors: {infra_n}") + print(f"(no non-skipped / non-error rows in {path})") + return 0 + print_table(summary, f"Summary: {path}") + return 0 + + if len(paths) == 2: + rows_a = load_rows(paths[0], dedupe=dedupe) + rows_b = load_rows(paths[1], dedupe=dedupe) + cmp = ab_compare(rows_a, rows_b) + print_ab_report(cmp, paths[0], paths[1]) + return 0 + + print( + "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/run.py b/evals/run.py new file mode 100644 index 00000000..7c5da868 --- /dev/null +++ b/evals/run.py @@ -0,0 +1,1203 @@ +"""CLI driver for the Plane MCP tool-surface eval harness. + +Usage: + python -m evals.run --list + python -m evals.run --dry-run --tasks R1 + python -m evals.run --tasks R1,W1,S1 --model sonnet --reps 1 --surface full +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.drivers import ( + KNOWN_DRIVERS, + agent_run_to_harness_dict, + get_driver, +) +from evals.seed import make_plane_client, seed, seed_plan, teardown +from evals.tasks import ( + TASKS, + PromptBindError, + TaskSkipped, + battery_fingerprint, + format_task_prompt, + get_tasks, + resolve_surface_tool_sets, + task_author, +) + +MODEL_ALIASES: dict[str, str] = { + "sonnet": "claude-sonnet-5", + "haiku": "claude-haiku-4-5", +} +# Per-driver resolution of the short harness aliases (sonnet/haiku). +# Drivers that need provider/model form get qualified defaults; unknown +# strings (e.g. ``anthropic/claude-…``) pass through unchanged. +CLI_MODEL_ALIASES: dict[str, dict[str, str]] = { + "claude-cli": {"sonnet": "sonnet", "haiku": "haiku"}, + "codex-cli": {"sonnet": "sonnet", "haiku": "haiku"}, + "antigravity-cli": { + "sonnet": "gemini-3.6-flash-high", + "haiku": "gemini-3.6-flash-low", + }, + "opencode-cli": { + "sonnet": "anthropic/claude-sonnet-4-20250514", + "haiku": "anthropic/claude-haiku-4-5-20251001", + }, +} + + +def resolve_model_for_driver(driver_name: str, model: str) -> str: + """Map a harness model token to the string the given driver expects. + + Known short aliases (sonnet/haiku) are looked up per-driver. Any other + string (including already-qualified ``provider/model``) is passed through. + """ + key = (driver_name or "sdk").strip().lower() + if key == "sdk": + return MODEL_ALIASES.get(model, model) + table = CLI_MODEL_ALIASES.get(key) or {} + return table.get(model, model) + + +# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). +# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env +# (see plane_mcp.v2.choose_stdio_mcp). +KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) + +DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "results" +MAX_ITERATIONS = 15 +MAX_TOKENS = 8192 + + +def _git_sha() -> str: + try: + return ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + cwd=Path(__file__).resolve().parent.parent, + ) + .decode() + .strip() + ) + except Exception: + return "unknown" + + +def _system_preamble(workspace_slug: str, project_name: str) -> str: + """Keep under 100 words — part of measured context.""" + return ( + f"You are evaluating Plane project management tools. " + f"Workspace slug: {workspace_slug}. Project name: {project_name}. " + f"Complete the task using the available tools, then stop." + ) + + +def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: + if tool in optimal: + return "optimal" + if tool in alternate: + return "alternate" + return "out_of_set" + + +def _tool_result_text(content: Any) -> tuple[str, str]: + """Return (text_for_counting, result_kind). + + result_kind is 'text' | 'image' | 'mixed'. Mixed keeps text for token counting + and records that non-text blocks were present (char length of full payload). + """ + if content is None: + return "", "text" + if isinstance(content, str): + return content, "text" + if isinstance(content, list): + texts: list[str] = [] + saw_non_text = False + for block in content: + btype = getattr(block, "type", None) or (block.get("type") if isinstance(block, dict) else None) + if btype == "text" or btype is None: + text = getattr(block, "text", None) or (block.get("text") if isinstance(block, dict) else None) + if text is None and isinstance(block, str): + text = block + if text is not None: + texts.append(str(text)) + else: + saw_non_text = True + joined = "\n".join(texts) + if texts and saw_non_text: + return joined, "mixed" + if texts: + return joined, "text" + if saw_non_text: + return json.dumps(content, default=str), "image" + return "", "text" + return str(content), "text" + + +async def _count_result_tokens(client: Any, model: str, result_text: str) -> int | None: + if not result_text: + return 0 + try: + n = await client.messages.count_tokens( + model=model, + messages=[{"role": "user", "content": result_text}], + ) + return n.input_tokens + except Exception as exc: + print(f"count_tokens warning: {exc}", file=sys.stderr) + return None + + +def _extract_final_text(message: Any) -> str: + if message is None: + return "" + parts: list[str] = [] + for block in getattr(message, "content", None) or []: + if getattr(block, "type", None) == "text" and getattr(block, "text", None): + parts.append(block.text) + return "\n".join(parts) + + +def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6). + + ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the + v2 tool registry. ``surface=full`` leaves the var unset (legacy default). + Other surface names (external servers under benchmark) set nothing; their + selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. + """ + env: dict[str, str] = {} + if path := os.environ.get("PATH"): + env["PATH"] = path + if home := os.environ.get("HOME"): + env["HOME"] = home + env["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] + env["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + env["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if surface == "v2": + env["PLANE_MCP_SURFACE"] = "v2" + elif surface == "v2-schema": + env["PLANE_MCP_SURFACE"] = "v2-schema" + if extra: + env.update(extra) + return env + + +def should_skip_resume_row(row: dict[str, Any]) -> bool: + """Return True if a prior row is a completed result that resume should skip. + + Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. + Rows with ``skipped`` set are treated as complete and are not retried (intentional: + surface/plan skips are stable outcomes, not infra failures). + Pure function — unit-tested without the live battery. + """ + ec = row.get("error_class") + if isinstance(ec, str) and ec.startswith("infra_"): + return False + if row.get("error") is not None: + return False + return True + + +def _resume_field_mismatch( + row: dict[str, Any], + *, + field: str, + expected: str | None, +) -> str | None: + """Return an error message if row[field] is present and disagrees with expected.""" + if expected is None: + return None + raw = row.get(field) + if raw is None or raw == "": + return None # back-compat: older rows without the key pass + # surface/driver compare case-insensitively; battery/model are exact strings. + if field in ("surface", "driver"): + got, want = str(raw).strip().lower(), expected.strip().lower() + else: + got, want = str(raw).strip(), expected.strip() + if got != want: + return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" + return None + + +def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: + """True when a CLI AgentRun stop_reason should be classified as infra_cli. + + ``timeout`` and Claude error subtypes (``error_during_execution``, bare + ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task + failure and stays in the success-rate denominator. + """ + if not stop_reason: + return False + sr = str(stop_reason) + if sr == "timeout": + return True + if sr == "error_max_turns": + return False + if sr == "error" or sr.startswith("error_"): + return True + return False + + +def _timeout_error_message(agent: dict[str, Any]) -> str: + """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" + for note in agent.get("driver_notes") or []: + if isinstance(note, str) and note.startswith("timeout after"): + return note + return "timeout" + + +def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: + """True for run-header meta lines or any row without a task_id.""" + if row.get("row_type") == "meta": + return True + return row.get("task_id") is None + + +def load_resume_skip_keys( + path: Path, + *, + surface: str, + battery: str | None = None, + model: str | None = None, + driver: str | None = None, +) -> tuple[set[tuple[str, int]], int, int]: + """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. + + Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` + (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / + battery / model / driver disagrees with the current run (missing keys pass for + back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked + but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. + """ + if not path.is_file(): + return set(), 0, 0 + skip_keys: set[tuple[str, int]] = set() + seen: set[tuple[str, int]] = set() + with path.open(encoding="utf-8") as fh: + for line_no, line in enumerate(fh, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: --resume {path}:{line_no}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict): + continue + for field, expected in ( + ("surface", surface), + ("battery", battery), + ("model", model), + ("driver", driver), + ): + msg = _resume_field_mismatch(row, field=field, expected=expected) + if msg: + raise SystemExit(msg) + # Meta / header rows: checked above, not part of resume key set. + if is_meta_or_non_task_row(row): + continue + tid = row.get("task_id") + rep = row.get("rep") + if tid is None or rep is None: + continue + key = (str(tid), int(rep)) + seen.add(key) + if should_skip_resume_row(row): + skip_keys.add(key) + else: + # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. + skip_keys.discard(key) + n_retry = len(seen - skip_keys) + return skip_keys, len(skip_keys), n_retry + + +def make_run_meta_row( + *, + run_id: str, + surface: str, + battery: str, + model: str | None, + driver: str, + git_sha: str, + ts: str | None = None, +) -> dict[str, Any]: + """Build the single first-line meta record for a new output JSONL.""" + return { + "row_type": "meta", + "run_id": run_id, + "surface": surface, + "battery": battery, + "model": model, + "driver": driver, + "git_sha": git_sha, + "ts": ts or datetime.now(timezone.utc).isoformat(), + } + + +def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: + """Write meta as the first line when the file is missing or empty. Returns True if written.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file() and path.stat().st_size > 0: + return False + with path.open("w", encoding="utf-8") as fh: + fh.write(json.dumps(meta, default=str) + "\n") + return True + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Plane MCP tool-surface eval harness") + p.add_argument("--list", action="store_true", help="Print task table (no network)") + p.add_argument("--dry-run", action="store_true", help="Print resolved prompts + seed plan (no network)") + p.add_argument("--tasks", type=str, default=None, help="Comma-separated task ids (default: all)") + p.add_argument( + "--model", + type=str, + default="sonnet", + help=( + "Model alias (sonnet/haiku) or a free-form provider/model id. " + "Short aliases are remapped per --driver (opencode/antigravity get qualified names)." + ), + ) + p.add_argument("--reps", type=int, default=1, help="Repetitions per task") + p.add_argument( + "--surface", + type=str, + default="full", + help=( + "Tool surface: 'full' (legacy 177 tools), 'v2', or 'v2-schema'. " + "With --server-cmd it is a free-form label for the external surface." + ), + ) + p.add_argument( + "--server-cmd", + type=str, + default=None, + help=( + "External MCP stdio server launch command (shlex-split), e.g. " + "'/path/venv/bin/python -m plane_mcp stdio --v2'. Enables external mode: " + "all tasks run (no surface skips) and mispick classification is disabled " + "(the foreign tool names have no overlay sets). CLI drivers only." + ), + ) + p.add_argument( + "--server-env", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra env var for the (external) MCP server child; repeatable.", + ) + p.add_argument( + "--driver", + type=str, + default="sdk", + choices=sorted(KNOWN_DRIVERS), + help=( + "Agent backend: sdk | claude-cli | codex-cli | antigravity-cli | opencode-cli. Not required for --canary." + ), + ) + p.add_argument("--out", type=str, default=None, help="JSONL output path") + p.add_argument( + "--resume", + type=str, + default=None, + metavar="OUT.jsonl", + help=( + "Resume into an existing JSONL (also the --out target). Skip (task_id, rep) " + "pairs that already completed; re-run rows with infra_ error_class or non-null error." + ), + ) + p.add_argument( + "--canary", + action="store_true", + help=( + "Verifier canary: seed each task, call verify with an empty agent result " + "(no driver/model), teardown. Exit 1 if any verifier returns ok=True on do-nothing." + ), + ) + return p.parse_args(argv) + + +def _task_ids(raw: str | None) -> list[str] | None: + if raw is None: + return None + return [t.strip() for t in raw.split(",") if t.strip()] + + +def cmd_list() -> int: + print(f"{'id':<6} {'tags':<18} {'opt':>4} prompt") + print("-" * 100) + for t in TASKS: + tags = ",".join(sorted(t["tags"])) + prompt = t["prompt"].replace("\n", " ") + if len(prompt) > 70: + prompt = prompt[:67] + "..." + print(f"{t['id']:<6} {tags:<18} {t['optimal_calls']:>4} {prompt}") + return 0 + + +def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: + needs: set[str] = set() + for t in tasks: + needs |= set(t.get("needs") or set()) + print("Seed plan:") + for line in seed_plan(needs): + print(f" {line}") + print() + sample_ctx = {"project_name": "EVAL deadbeef"} + for t in tasks: + resolved = format_task_prompt(t, sample_ctx, strict=False) + print(f"=== {t['id']} ===") + print(f"needs: {sorted(t.get('needs') or [])}") + print(f"author: {t.get('author') or 'claude'}") + print(f"optimal_calls: {t['optimal_calls']}") + print(f"optimal_tools: {sorted(t['optimal_tools'])}") + print(f"prompt:\n {resolved}") + print() + return 0 + + +async def run_agent_task( + *, + client: Any, + model_id: str, + task: dict[str, Any], + ctx: dict[str, Any], + workspace_slug: str, + surface: str = "full", + optimal_tools: set[str] | None = None, + alternate_tools: set[str] | None = None, +) -> dict[str, Any]: + """Run one task against a fresh stdio MCP server subprocess.""" + from anthropic.lib.tools.mcp import async_mcp_tool + from mcp import ClientSession + from mcp.client.stdio import StdioServerParameters, stdio_client + + project_name = ctx["project_name"] + system = _system_preamble(workspace_slug, project_name) + # strict: empty binder values / exceptions are infra_seed, not blank-ID prompts. + prompt = format_task_prompt(task, ctx, strict=True) + + server_params = StdioServerParameters( + command=sys.executable, + args=["-m", "plane_mcp", "stdio"], + env=stdio_server_env(surface=surface), + ) + + optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) + alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) + assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" + + calls: list[dict[str, Any]] = [] + # (call_idx, text, kind, is_error) buffered for post-loop count_tokens + pending_results: list[tuple[int, str, str, bool]] = [] + usage_per_iteration: list[dict[str, int]] = [] + final_message = None + iterations = 0 + result_pair_mismatch = False + wall_time_s = 0.0 + + async with stdio_client(server_params) as (read, write): + async with ClientSession(read, write) as mcp_client: + await mcp_client.initialize() + tools_result = await mcp_client.list_tools() + runner = client.beta.messages.tool_runner( + model=model_id, + max_tokens=MAX_TOKENS, + max_iterations=MAX_ITERATIONS, + system=system, + messages=[{"role": "user", "content": prompt}], + tools=[async_mcp_tool(t, mcp_client) for t in tools_result.tools], + ) + # wall_time: agent loop only (after list_tools, before subprocess teardown) + t0 = time.perf_counter() + try: + async for message in runner: + iterations += 1 + final_message = message + usage = getattr(message, "usage", None) + if usage is not None: + usage_per_iteration.append( + { + "in": getattr(usage, "input_tokens", 0) or 0, + "out": getattr(usage, "output_tokens", 0) or 0, + "cache_read": getattr(usage, "cache_read_input_tokens", 0) or 0, + "cache_write": getattr(usage, "cache_creation_input_tokens", 0) or 0, + } + ) + + # Map tool_use_id → call index for result pairing (not ordinal-only). + tool_use_by_id: dict[str, int] = {} + for block in message.content or []: + if getattr(block, "type", None) == "tool_use": + name = block.name + args = block.input if hasattr(block, "input") else {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + call_rec = { + "tool": name, + "class": classify_call(name, optimal, alternate), + "args_chars": args_chars, + "result_tokens": None, + "result_chars": 0, + "result_kind": "text", + "is_error": False, + } + idx = len(calls) + calls.append(call_rec) + use_id = getattr(block, "id", None) + if use_id: + tool_use_by_id[str(use_id)] = idx + + # Never execute tools on a refusal-terminated turn (F2 / SDK guard). + if getattr(message, "stop_reason", None) == "refusal": + continue + + tool_response = await runner.generate_tool_call_response() + if tool_response is not None: + if isinstance(tool_response, dict): + blocks = tool_response.get("content") or [] + else: + blocks = getattr(tool_response, "content", None) or [] + result_blocks = [ + b + for b in blocks + if getattr(b, "type", None) == "tool_result" + or (isinstance(b, dict) and b.get("type") == "tool_result") + ] + matched_ids: set[str] = set() + for block in result_blocks: + if isinstance(block, dict): + is_error = bool(block.get("is_error")) + raw_content = block.get("content") + tool_use_id = block.get("tool_use_id") + else: + is_error = bool(getattr(block, "is_error", False)) + raw_content = getattr(block, "content", None) + tool_use_id = getattr(block, "tool_use_id", None) + text, kind = _tool_result_text(raw_content) + if tool_use_id is not None and str(tool_use_id) in tool_use_by_id: + idx = tool_use_by_id[str(tool_use_id)] + matched_ids.add(str(tool_use_id)) + else: + result_pair_mismatch = True + continue + calls[idx]["is_error"] = is_error + calls[idx]["result_kind"] = kind + if kind == "text": + calls[idx]["result_chars"] = len(text) + else: + calls[idx]["result_chars"] = len(str(raw_content)) + pending_results.append((idx, text, kind, is_error)) + if len(matched_ids) != len(tool_use_by_id): + result_pair_mismatch = True + finally: + wall_time_s = time.perf_counter() - t0 + + # Token-count tool results after the agent loop (must not pollute wall_time). + token_count_failures = 0 + for idx, text, kind, _is_error in pending_results: + if kind not in ("text", "mixed"): + calls[idx]["result_tokens"] = None + continue + counted = await _count_result_tokens(client, model_id, text) + calls[idx]["result_tokens"] = counted + if counted is None and text: + token_count_failures += 1 + + stop_reason = getattr(final_message, "stop_reason", None) if final_message else None + # Cap detection is stop_reason-aware only (F0/F3): a clean end_turn (or max_tokens, + # which the report already counts separately) on the 15th yield is not flagged here. + # Only runs that exhaust the iteration budget while still mid-tool-loop count. + hit_max_iterations = iterations >= MAX_ITERATIONS and stop_reason not in ( + "end_turn", + "max_tokens", + ) + + final_text = _extract_final_text(final_message) + errored = sum(1 for c in calls if c.get("is_error")) + alternate_n = sum(1 for c in calls if c["class"] == "alternate") + out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") + total_result_tokens = sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None) + cum_input = sum(u.get("in", 0) for u in usage_per_iteration) + + return { + "final_text": final_text, + "calls": calls, + "num_calls": len(calls), + "errored_calls": errored, + "alternate_calls": alternate_n, + "out_of_set_calls": out_of_set_n, + "total_result_tokens": total_result_tokens, + "usage_per_iteration": usage_per_iteration, + "cum_input_tokens": cum_input, + "wall_time_s": round(wall_time_s, 3), + "stop_reason": stop_reason, + "hit_max_iterations": hit_max_iterations, + "result_pair_mismatch": result_pair_mismatch, + "token_count_failures": token_count_failures, + } + + +async def run_agent_task_via_driver( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + ctx: dict[str, Any], + workspace_slug: str, + surface: str = "full", + optimal_tools: set[str] | None = None, + alternate_tools: set[str] | None = None, + server_env: dict[str, str] | None = None, +) -> dict[str, Any]: + """Run one task through a CLI (or other) AgentDriver.""" + project_name = ctx["project_name"] + system = _system_preamble(workspace_slug, project_name) + prompt = format_task_prompt(task, ctx, strict=True) + optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) + alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) + assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" + + mcp_env = stdio_server_env(surface=surface, extra=server_env) + # Drivers are sync (subprocess); run off the event loop thread. + agent_run = await asyncio.to_thread( + driver.run_task, + prompt, + mcp_env, + model_id, + MAX_ITERATIONS, + system=system, + cwd=Path(__file__).resolve().parent.parent, + ) + return agent_run_to_harness_dict( + agent_run, + optimal=optimal, + alternate=alternate, + classify=classify_call, + skip_result_tokens=True, + ) + + +def _base_row( + *, + run_id: str, + git_sha: str, + surface: str, + driver_name: str, + model_id: str | None, + task: dict[str, Any], + rep: int, + battery: str, + classification: str, +) -> dict[str, Any]: + return { + "run_id": run_id, + "ts": datetime.now(timezone.utc).isoformat(), + "git_sha": git_sha, + "battery": battery, + "surface": surface, + "driver": driver_name, + "classification": classification, + "model": model_id, + "task_id": task["id"], + "author": task_author(task), + "rep": rep, + "success": False, + "verify_note": "", + "skipped": None, + "error": None, + "error_class": None, + "stop_reason": None, + "hit_max_iterations": False, + "calls": [], + "num_calls": 0, + "errored_calls": 0, + "alternate_calls": 0, + "out_of_set_calls": 0, + "total_result_tokens": 0, + "usage_per_iteration": [], + "cum_input_tokens": 0, + "wall_time_s": 0.0, + } + + +async def run_live( + tasks: list[dict[str, Any]], + *, + model_alias: str, + reps: int, + surface: str, + out_path: Path, + driver_name: str = "sdk", + server_cmd: list[str] | None = None, + server_env: dict[str, str] | None = None, + resume: bool = False, +) -> int: + surface = (surface or "full").strip().lower() + external = server_cmd is not None + if not external and surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " + "(or pass --server-cmd for an external surface)", + file=sys.stderr, + ) + return 2 + + driver_name = (driver_name or "sdk").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + if external and driver_name == "sdk": + print( + f"error: --server-cmd requires a CLI driver (one of {sorted(KNOWN_DRIVERS - {'sdk'})})", + file=sys.stderr, + ) + return 2 + + use_sdk = driver_name == "sdk" + model_id = resolve_model_for_driver(driver_name, model_alias) + + run_id = uuid.uuid4().hex + git_sha = _git_sha() + battery = battery_fingerprint(tasks) + out_path.parent.mkdir(parents=True, exist_ok=True) + + resume_skip: set[tuple[str, int]] = set() + if resume: + try: + resume_skip, n_skip, n_retry = load_resume_skip_keys( + out_path, + surface=surface, + battery=battery, + model=model_id, + driver=driver_name, + ) + except SystemExit as e: + print(e, file=sys.stderr) + return 2 + print(f"resume: skipping {n_skip} completed rows, retrying {n_retry}") + + # First line of a new/empty file is a meta header (skipped by loaders). + meta = make_run_meta_row( + run_id=run_id, + surface=surface, + battery=battery, + model=model_id, + driver=driver_name, + git_sha=git_sha, + ) + if maybe_write_run_meta(out_path, meta): + print(f"wrote meta header battery={battery} surface={surface}") + + plane, workspace_slug = make_plane_client() + # User chose --driver explicitly: codex live is allowed (they own the quota). + driver_kwargs: dict[str, Any] = {} + if driver_name == "codex-cli": + driver_kwargs["allow_live"] = True + # --server-cmd must reach every CLI driver (not just Claude); otherwise we + # silently benchmark the wrong server. + if server_cmd is not None: + if use_sdk: + print("error: --server-cmd is incompatible with --driver sdk", file=sys.stderr) + return 2 + driver_kwargs["server_command"] = server_cmd + cli_driver = None if use_sdk else get_driver(driver_name, **driver_kwargs) + + print( + f"run_id={run_id} battery={battery} driver={driver_name} model={model_id} " + f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" + ) + print(f"writing {out_path}") + + async def _one_client_scope(client: Any | None) -> None: + with out_path.open("a", encoding="utf-8") as fh: + for task in tasks: + if external: + # Foreign tool names have no overlay sets: no skips, no + # mispick classification — success/calls/errors only. + surface_sets = { + "skip": None, + "optimal_tools": set(), + "alternate_tools": set(), + "classification": "external", + } + else: + surface_sets = resolve_surface_tool_sets(task, surface) + for rep in range(reps): + if (task["id"], rep) in resume_skip: + print(f" {task['id']} rep={rep} RESUME_SKIP") + continue + + ctx: dict[str, Any] = {} + row = _base_row( + run_id=run_id, + git_sha=git_sha, + surface=surface, + driver_name=driver_name, + model_id=model_id, + task=task, + rep=rep, + battery=battery, + classification=str(surface_sets["classification"]), + ) + try: + # Surface-unsupported tasks: record skip, no seed/agent. + if surface_sets.get("skip"): + reason = surface_sets["skip"] + row["skipped"] = reason + row["verify_note"] = reason + print(f" {task['id']} rep={rep} SKIPPED: {reason}") + else: + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) + except TaskSkipped as skip: + row["skipped"] = skip.reason + row["verify_note"] = skip.reason + print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") + except Exception as exc: + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "infra_seed" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + if ctx.get("project_name"): + print( + f" orphaned project may remain: {ctx['project_name']}", + file=sys.stderr, + ) + else: + if "bug_type" in task_needs and not ctx.get("bug_type"): + reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" + row["skipped"] = reason + row["verify_note"] = reason + print(f" {task['id']} rep={rep} SKIPPED: {reason}") + else: + agent: dict[str, Any] | None = None + # Agent wrap: SDK bugs → infra_sdk; CLI raises → infra_cli. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + if use_sdk: + assert client is not None + agent = await run_agent_task( + client=client, + model_id=model_id, + task=task, + ctx=ctx, + workspace_slug=workspace_slug, + surface=surface, + optimal_tools=surface_sets["optimal_tools"], + alternate_tools=surface_sets["alternate_tools"], + ) + else: + assert cli_driver is not None + agent = await run_agent_task_via_driver( + driver=cli_driver, + model_id=model_id, + task=task, + ctx=ctx, + workspace_slug=workspace_slug, + surface=surface, + optimal_tools=surface_sets["optimal_tools"], + alternate_tools=surface_sets["alternate_tools"], + server_env=server_env, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "infra_seed" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + agent = None + except Exception as exc: + # SDK harness bugs must not look like CLI infra. + agent_err_class = "infra_sdk" if use_sdk else "infra_cli" + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = agent_err_class + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[{agent_err_class}]: {exc}", + file=sys.stderr, + ) + agent = None + + if agent is not None: + row.update({k: agent[k] for k in agent if k != "final_text"}) + if external: + # Empty overlay sets would classify every call + # out-of-set; null the counters instead. + row["alternate_calls"] = None + row["out_of_set_calls"] = None + + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.get("stop_reason") + if not use_sdk and is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): + row["success"] = False + row["error_class"] = "infra_cli" + if stop_reason == "timeout": + row["error"] = _timeout_error_message(agent) + else: + notes = [ + n for n in (agent.get("driver_notes") or []) if isinstance(n, str) + ] + detail = "; ".join(notes) if notes else str(stop_reason) + row["error"] = detail + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_cli]: {row['error']}", + file=sys.stderr, + ) + else: + verify = task["verify"] + try: + ok, note = await verify( + plane, + ctx, + { + "final_text": agent["final_text"], + "calls": agent["calls"], + }, + ) + row["success"] = bool(ok) + row["verify_note"] = note + print( + f" {task['id']} rep={rep} success={ok} " + f"calls={agent['num_calls']} note={note!r}" + ) + except TaskSkipped as skip: + row["skipped"] = skip.reason + row["verify_note"] = skip.reason + print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") + except Exception as exc: + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "task" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[task]: {exc}", + file=sys.stderr, + ) + except Exception as exc: + # Anything outside seed/driver/verify wraps. + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "task" + row["verify_note"] = "" + print(f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr) + if ctx.get("project_name"): + print( + f" orphaned project may remain: {ctx['project_name']}", + file=sys.stderr, + ) + finally: + try: + teardown(plane, ctx) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + if ctx.get("project_name"): + print(f" orphaned project: {ctx['project_name']}", file=sys.stderr) + + fh.write(json.dumps(row, default=str) + "\n") + fh.flush() + + if use_sdk: + from anthropic import AsyncAnthropic + + async with AsyncAnthropic() as client: + await _one_client_scope(client) + else: + await _one_client_scope(None) + + return 0 + + +async def run_canary( + tasks: list[dict[str, Any]], + *, + surface: str, +) -> int: + """Seed + verify(empty agent) + teardown per task; no driver/model. + + Passes only when every verifier returns falsy ok on a do-nothing agent. + Any ok=True is a broken verifier (false positive). + """ + surface = (surface or "full").strip().lower() + if surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", + file=sys.stderr, + ) + return 2 + + battery = battery_fingerprint(tasks) + plane, _workspace_slug = make_plane_client() + print(f"canary battery={battery} surface={surface} tasks={[t['id'] for t in tasks]}") + + broken: list[str] = [] + verified_count = 0 + empty_agent = {"final_text": "", "calls": []} + + for task in tasks: + surface_sets = resolve_surface_tool_sets(task, surface) + if surface_sets.get("skip"): + print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") + continue + + ctx: dict[str, Any] = {} + task_needs = set(task.get("needs") or set()) + try: + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + if "bug_type" in task_needs and not ctx.get("bug_type"): + reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" + print(f" {task['id']} SKIPPED: {reason}") + continue + try: + ok, note = await task["verify"](plane, ctx, empty_agent) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + verified_count += 1 + if ok: + broken.append(task["id"]) + print(f" BROKEN VERIFIER: {task['id']} note={note!r}") + else: + print(f" {task['id']} ok=False note={note!r}") + except Exception as exc: + print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) + broken.append(task["id"]) + finally: + try: + teardown(plane, ctx) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + + if broken: + for tid in broken: + print(f"BROKEN VERIFIER: {tid}", file=sys.stderr) + return 1 + if verified_count == 0: + print( + "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", + file=sys.stderr, + ) + return 1 + print(f"canary: all verifiers reject empty agent ({verified_count} verified)") + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + + if args.list: + return cmd_list() + + ids = _task_ids(args.tasks) + try: + tasks = get_tasks(ids) + except SystemExit as e: + print(e, file=sys.stderr) + return 2 + + if args.dry_run: + return cmd_dry_run(tasks) + + surface = (args.surface or "full").strip().lower() + server_cmd: list[str] | None = None + if args.server_cmd: + import shlex + + server_cmd = shlex.split(args.server_cmd) + if not server_cmd: + print("error: --server-cmd is empty", file=sys.stderr) + return 2 + elif surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " + "(or pass --server-cmd for an external surface)", + file=sys.stderr, + ) + return 2 + + server_env: dict[str, str] = {} + for pair in args.server_env: + key, sep, val = pair.partition("=") + if not sep or not key: + print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) + return 2 + server_env[key] = val + + # Canary: live env only — no driver/model required. + if args.canary: + return asyncio.run(run_canary(tasks, surface=surface)) + + driver_name = (getattr(args, "driver", None) or "sdk").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + + if args.resume: + out = Path(args.resume) + elif args.out: + out = Path(args.out) + else: + out = DEFAULT_OUT_DIR / f"{uuid.uuid4().hex}.jsonl" + + return asyncio.run( + run_live( + tasks, + model_alias=args.model, + reps=args.reps, + surface=surface, + out_path=out, + driver_name=driver_name, + server_cmd=server_cmd, + server_env=server_env or None, + resume=bool(args.resume), + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/seed.py b/evals/seed.py new file mode 100644 index 00000000..891e6417 --- /dev/null +++ b/evals/seed.py @@ -0,0 +1,1152 @@ +"""Per-run fixture create/teardown via plane-sdk.""" + +from __future__ import annotations + +import os +import secrets +from datetime import date, timedelta +from typing import Any + +from plane import PlaneClient +from plane.errors.errors import HttpError +from plane.models.customers import CreateCustomer, CreateCustomerRequest +from plane.models.cycles import CreateCycle, UpdateCycle +from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest +from plane.models.labels import CreateLabel +from plane.models.modules import CreateModule +from plane.models.projects import CreateProject, ProjectFeature, UpdateProject +from plane.models.releases import CreateRelease, UpdateReleaseChangelog +from plane.models.work_item_types import CreateWorkItemType +from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem +from plane.models.workspaces import WorkspaceFeature + +# Soft-deleted projects reserve identifiers; create may 409 — retry with a new suffix. +_PROJECT_CREATE_MAX_ATTEMPTS = 3 + +# Fixed fixture titles for the `items` group. Exactly 4 urgent; the rest medium/high/low. +# "Payment webhook drops retries" is the R1 target (urgent, non-default state). +ITEM_FIXTURES: list[tuple[str, str]] = [ + ("Payment webhook drops retries", "urgent"), + ("Checkout times out on 3DS challenge", "urgent"), + ("Session cookie not rotated after login", "urgent"), + ("Inventory count goes negative under load", "urgent"), + ("Search results ignore archived projects", "high"), + ("CSV export truncates multi-byte chars", "high"), + ("Webhook secret rotation docs missing", "medium"), + ("Dark mode contrast fails WCAG AA", "medium"), + ("Onboarding email template stale", "medium"), + ("Sidebar collapse flickers on resize", "low"), + ("Tooltip clipped inside modal dialog", "low"), + ("Footer year still says 2024", "none"), +] + +R1_TITLE = ITEM_FIXTURES[0][0] +# R5 discussion target + distinctive comment phrases (word-boundary matched at verify). +R5_TITLE = "Checkout times out on 3DS challenge" +R5_COMMENT_PHRASES = ( + "stripe callback race", + "retry budget exhausted", +) +# W2 / W3 / W8 targets +W2_TITLE = "Sidebar collapse flickers on resize" +W3_TITLE = "Dark mode contrast fails WCAG AA" +W8_TITLE = R1_TITLE +# W7 relation pair + reference URL +W7_SOURCE_TITLE = "Search results ignore archived projects" +W7_TARGET_TITLE = "CSV export truncates multi-byte chars" +W7_URL = "https://example.com/eval/runbook-w7" +# R3: assignees + due this week (seeded count stored in ctx) +R3_DUE_TITLES = ( + "Webhook secret rotation docs missing", + "Onboarding email template stale", +) +# W6 unfinished items in Sprint 12 +W6_UNFINISHED_TITLES = ( + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", +) +# Module completed-item titles (created extra when seeding module) +MODULE_NAME = "Checkout revamp" +MODULE_COMPLETED_TITLES = ( + "Module done: cart totals", + "Module done: tax lines", + "Module done: shipping quote", +) +# Intake fixtures +INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" +INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" +# Customer / release +CUSTOMER_NAME = "Acme Corp" +CUSTOMER_REQUEST_NAME = "SSO support" +RELEASE_NAME = "1.2.0" +RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +# R6 second project bug titles +R6_MAIN_BUG_TITLES = ("Main bug alpha", "Main bug beta") +R6_SECOND_BUG_TITLES = ( + "Second bug one", + "Second bug two", + "Second bug three", + "Second bug four", +) + +LABEL_NAMES = ("auth", "triage", "perf") +CYCLE_PAST = "Sprint 12" +CYCLE_CURRENT = "Sprint 13" +# WS3 long-tail fixtures that live at workspace scope (must pre-clean + teardown). +DEBIAS_RELEASE_TAG_VERSION = "eval-rc1" +DEBIAS_CUSTOMER_PROP_DISPLAY = "Eval Industry" + + +def make_plane_client() -> tuple[PlaneClient, str]: + """Build a PlaneClient from EVAL_* env vars (mirrors stdio client construction).""" + api_key = os.environ.get("EVAL_PLANE_API_KEY", "") + workspace_slug = os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + base_url = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if not api_key or not workspace_slug: + raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for live runs") + client = PlaneClient(base_url=base_url, api_key=api_key) + return client, workspace_slug + + +def seed_plan(needs: set[str]) -> list[str]: + """Human-readable seed plan for --dry-run (no network).""" + lines = [ + "project: EVAL {run8} (identifier EV{XXXX})", + ] + if "items" in needs: + lines.append(f"items: {len(ITEM_FIXTURES)} work items (exactly 4 urgent open)") + lines.append(f" - {R1_TITLE!r} (urgent, non-default started-group state) # R1 target") + lines.append(f" - {len(R3_DUE_TITLES)} assigned-to-me with due this week # R3") + lines.append(f" - comments on {R5_TITLE!r} # R5 discussion") + if "activity_feed" in needs: + lines.append( + f"activity_feed: gate that activities exist for {R5_TITLE!r} " + "(TaskSkipped env:no-activity-worker if empty) # L2" + ) + if "labels" in needs: + lines.append(f"labels: {', '.join(LABEL_NAMES)}") + if "bug_type" in needs: + lines.append( + "bug_type: work item type 'Bug' (genuine plan-gate only → skip dependents; other seed errors raise)" + ) + if "cycles" in needs: + past_state = "ends tomorrow, still OPEN so it can be closed" if "cycles_open_past" in needs else "past-dated" + lines.append(f"cycles: {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); unfinished on past") + if "module" in needs: + lines.append(f"module: {MODULE_NAME!r} with {len(MODULE_COMPLETED_TITLES)} completed items") + if "intake" in needs: + lines.append(f"intake: billing {INTAKE_BILLING_TITLE!r} + spam {INTAKE_SPAM_TITLE!r}") + if "customer" in needs: + lines.append(f"customer: {CUSTOMER_NAME!r} + request {CUSTOMER_REQUEST_NAME!r}") + if "release" in needs: + lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") + if "second_project" in needs: + lines.append("second_project: EVAL {run8} B with more open Bug-typed items than main (R6)") + if "leave_cycles_worklogs_off" in needs: + lines.append( + "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " + "(agent enables; teardown re-enables customers=True for later C1)" + ) + else: + lines.append( + "workspace_features: customers=True " + "(is_customer_enabled; NOT work_item_types — leaves S1/S3 type mode alone)" + ) + return lines + + +def _is_plan_gate(exc: BaseException) -> bool: + """True only for genuine plan/subscription feature gates — not generic API failures.""" + if not isinstance(exc, HttpError): + return False + if exc.status_code in (402, 403): + return True + blob = f"{exc} {exc.response!s}".lower() + keywords = ("plan", "subscription", "upgrade", "not available on your", "feature is not enabled") + return any(k in blob for k in keywords) + + +def is_identifier_collision(exc: BaseException) -> bool: + """True when project create failed because the identifier is already taken. + + Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare + ``identifier`` mention (validation shape errors) must not trigger retry. + """ + if not isinstance(exc, HttpError): + return False + if exc.status_code not in (400, 409): + return False + blob = f"{exc} {exc.response!s}".lower() + return any(k in blob for k in ("already", "exists", "taken")) + + +def create_project_with_identifier_retry( + plane: PlaneClient, + workspace_slug: str, + *, + name: str, + identifier_prefix: str, + initial_suffix: str, +) -> Any: + """Create a project, regenerating the identifier suffix on soft-delete collisions. + + Plane soft-deletes reserve identifiers; a 409 (or identifier-in-message error) + triggers a new random 4-char hex suffix. Max ``_PROJECT_CREATE_MAX_ATTEMPTS`` + attempts, then re-raises the last collision error. + """ + suffix = (initial_suffix or "")[:4].upper() + if len(suffix) < 4: + suffix = (suffix + secrets.token_hex(2).upper())[:4] + last_exc: BaseException | None = None + for attempt in range(_PROJECT_CREATE_MAX_ATTEMPTS): + if attempt > 0: + suffix = secrets.token_hex(2).upper() # 4 hex chars + identifier = f"{identifier_prefix}{suffix}" + try: + return plane.projects.create( + workspace_slug=workspace_slug, + data=CreateProject(name=name, identifier=identifier), + ) + except Exception as exc: + if is_identifier_collision(exc): + last_exc = exc + continue + raise + if last_exc is None: + raise RuntimeError( + f"project create failed after {_PROJECT_CREATE_MAX_ATTEMPTS} identifier retries " + f"(prefix={identifier_prefix!r}) with no captured exception" + ) + raise last_exc + + +def _enable_workspace_features( + plane: PlaneClient, + workspace_slug: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> None: + """Enable workspace-level feature toggles that task preconditions need. + + Gate (plane-ee): create-customer 403 when + ``check_workspace_feature(slug, IS_CUSTOMER_ENABLED)`` is false — DB column + ``WorkspaceFeature.is_customer_enabled``. Legacy/SDK flips it via + ``workspaces.update_features`` / ``WorkspaceFeature(customers=True)`` + (API serializer maps ``customers`` → ``is_customer_enabled``). + + Deliberately does **not** set ``work_item_types``: that flips + workspace-vs-project type ownership and would change S1/S3 seed mode. + + ``exclude`` may contain ``customers`` (S5 leaves it off for the agent to enable). + """ + skip = set(exclude or ()) + data: dict[str, bool] = {} + if "customers" not in skip: + data["customers"] = True + if not data: + return + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(**data), + ) + + +def _enable_project_features( + plane: PlaneClient, + workspace_slug: str, + project_id: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> None: + """Enable per-project feature gates that fresh projects ship with disabled. + + Two SDK calls (harmless if already on): + + 1. ``projects.update`` / ``UpdateProject`` — view columns API gates read: + ``cycle_view``, ``module_view``, ``intake_view``, ``page_view``, + ``is_time_tracking_enabled`` (worklog 404 when false). + 2. ``projects.update_features`` / ``ProjectFeature`` — capability flags + that the server maps onto the same view columns for cycles/modules/… . + + ``exclude`` is a set of feature keys to leave disabled (for S5): + ``cycles``, ``modules``, ``intakes``, ``pages``, ``worklogs``. + Default: enable all (other catalog tasks need them). + """ + skip = set(exclude or ()) + + upd_kwargs: dict[str, bool] = {} + if "cycles" not in skip: + upd_kwargs["cycle_view"] = True + if "modules" not in skip: + upd_kwargs["module_view"] = True + if "intakes" not in skip: + upd_kwargs["intake_view"] = True + if "pages" not in skip: + upd_kwargs["page_view"] = True + if "worklogs" not in skip: + upd_kwargs["is_time_tracking_enabled"] = True + if upd_kwargs: + plane.projects.update( + workspace_slug=workspace_slug, + project_id=project_id, + data=UpdateProject(**upd_kwargs), + ) + + feat_kwargs: dict[str, bool] = {} + if "cycles" not in skip: + feat_kwargs["cycles"] = True + if "modules" not in skip: + feat_kwargs["modules"] = True + if "intakes" not in skip: + feat_kwargs["intakes"] = True + if "pages" not in skip: + feat_kwargs["pages"] = True + if feat_kwargs: + plane.projects.update_features( + workspace_slug=workspace_slug, + project_id=project_id, + data=ProjectFeature(**feat_kwargs), + ) + + +def _preclean_ws3_workspace_artifacts(plane: PlaneClient, workspace_slug: str) -> None: + """Delete leftover WS3 long-tail artifacts so a dirty workspace cannot false-pass. + + Removes any existing release tag ``eval-rc1`` and customer property + ``Eval Industry`` before the rep seeds. + + Empty / not-found lists are silent. Clients without the API surface (offline + test stubs) are skipped silently. If a matching artifact is **found** and + cannot be deleted — or list fails on a present API — raises so the harness + records ``infra_seed`` rather than running against dirty state. + """ + releases = getattr(plane, "releases", None) + tags_api = getattr(releases, "tags", None) if releases is not None else None + if tags_api is not None: + try: + page = tags_api.list(workspace_slug=workspace_slug) + except Exception as exc: + raise RuntimeError(f"WS3 preclean: list release tags failed: {exc}") from exc + rows = page.results if hasattr(page, "results") else page + for tag in rows or []: + ver = (getattr(tag, "version", None) or "").strip() + if ver != DEBIAS_RELEASE_TAG_VERSION: + continue + tid = getattr(tag, "id", None) + if not tid: + continue + try: + tags_api.delete(workspace_slug=workspace_slug, tag_id=tid) + except Exception as exc: + raise RuntimeError( + f"WS3 preclean: failed to delete stale release tag {DEBIAS_RELEASE_TAG_VERSION!r} id={tid}: {exc}" + ) from exc + + customers = getattr(plane, "customers", None) + props_api = getattr(customers, "properties", None) if customers is not None else None + if props_api is not None: + try: + page = props_api.list(workspace_slug=workspace_slug) + except Exception as exc: + raise RuntimeError(f"WS3 preclean: list customer properties failed: {exc}") from exc + rows = page.results if hasattr(page, "results") else page + target = DEBIAS_CUSTOMER_PROP_DISPLAY.casefold() + for prop in rows or []: + display = (getattr(prop, "display_name", None) or getattr(prop, "name", None) or "").strip() + if display.casefold() != target: + continue + pid = getattr(prop, "id", None) + if not pid: + continue + try: + props_api.delete(workspace_slug=workspace_slug, property_id=pid) + except Exception as exc: + raise RuntimeError( + f"WS3 preclean: failed to delete stale customer property " + f"{DEBIAS_CUSTOMER_PROP_DISPLAY!r} id={pid}: {exc}" + ) from exc + + +def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) -> dict[str, Any]: + """Create the eval project and declared fixture groups. + + Mutates the caller-provided `ctx` in place so project_id is visible to teardown + even if a later fixture step raises (F5). + """ + run8 = run_id[:8] + project_name = f"EVAL {run8}" + workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + + # Defensive: drop WS3 workspace artifacts that would make a no-op agent pass. + _preclean_ws3_workspace_artifacts(plane, workspace_slug) + + # Reset known keys while preserving object identity for the caller. + ctx.clear() + ctx.update( + { + "run_id": run_id, + "run8": run8, + "workspace_slug": workspace_slug, + "project_id": None, + "project_name": project_name, + "project_identifier": None, # filled after create (may retry suffix) + "labels": {}, + "items": {}, + "item_identifiers": {}, # title -> PROJ-N for ID-in-hand prompts + "item_ids": [], + "state_names": [], # all project state display names (for R1 negative check) + "r1_state_name": None, + "bug_type": None, + "bug_type_created": False, + "bug_type_workspace_level": False, + "bug_type_skip_reason": None, + "cycles": {}, + "module": None, + "module_completed_ids": [], + "intake": {}, + "customer": None, + "customer_request": None, + "release": None, + "second_project_id": None, + "second_project_name": None, + "r3_due_titles": list(R3_DUE_TITLES), + "r3_due_count": len(R3_DUE_TITLES), + "r5_title": R5_TITLE, + "r5_comment_phrases": list(R5_COMMENT_PHRASES), + "w6_unfinished_titles": list(W6_UNFINISHED_TITLES), + "workspace_objects": [], # [{kind, id}, ...] surviving project delete + } + ) + + # EV + 4 hex chars; retry with a new suffix on soft-delete identifier collisions. + project = create_project_with_identifier_retry( + plane, + workspace_slug, + name=project_name, + identifier_prefix="EV", + initial_suffix=run8[:4].upper(), + ) + ctx["project_id"] = project.id + ctx["project_identifier"] = getattr(project, "identifier", None) + + # Feature enablement (workspace first, then project). + # + # Ordering for S5 vs C1 on a shared eval workspace: + # - Each task-rep has its own seed/teardown; there is no multi-task seed batch. + # - Default tasks: enable workspace customers=True so C1 create_customer works. + # - S5 (needs leave_cycles_worklogs_off): leave project cycles+worklogs AND + # workspace customers OFF so the agent must flip all three; teardown then + # re-enables customers=True so a later C1 rep is not left 403ing. + # - We do not try to "run workspace enable after S5 check" — seed is per-task. + feature_exclude: set[str] = set() + ws_feature_exclude: set[str] = set() + if "leave_cycles_worklogs_off" in needs: + feature_exclude = {"cycles", "worklogs"} + ws_feature_exclude = {"customers"} + ctx["s5_left_customers_off"] = True + ctx["feature_exclude"] = sorted(feature_exclude) + ctx["ws_feature_exclude"] = sorted(ws_feature_exclude) + _enable_workspace_features(plane, workspace_slug, exclude=ws_feature_exclude) + _enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) + + # Labels before items so items can attach labels later if needed. + if "labels" in needs: + _seed_labels(plane, workspace_slug, ctx) + if "items" in needs: + _seed_items(plane, workspace_slug, ctx) + # L2: comments must materialize as activities (activity worker must be running). + if "activity_feed" in needs: + if "items" not in needs and not ctx.get("item_ids"): + _seed_items(plane, workspace_slug, ctx) + _gate_activity_worker(plane, workspace_slug, ctx) + if "bug_type" in needs: + _seed_bug_type(plane, workspace_slug, ctx) + if "cycles" in needs: + # Cycles need items to attach unfinished work; seed items if not already. + if "items" not in needs and not ctx["item_ids"]: + _seed_items(plane, workspace_slug, ctx) + _seed_cycles(plane, workspace_slug, ctx, leave_past_open="cycles_open_past" in needs) + if "module" in needs: + _seed_module(plane, workspace_slug, ctx) + if "intake" in needs: + _seed_intake(plane, workspace_slug, ctx) + if "customer" in needs: + _seed_customer(plane, workspace_slug, ctx) + if "release" in needs: + _seed_release(plane, workspace_slug, ctx) + if "second_project" in needs: + _seed_second_project(plane, workspace_slug, ctx) + + return ctx + + +def _seed_labels(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + for name in LABEL_NAMES: + label = plane.labels.create( + workspace_slug=workspace_slug, + project_id=ctx["project_id"], + data=CreateLabel(name=name), + ) + ctx["labels"][name] = label.id + + +def _list_states(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + return list(page.results or []) + + +def _completed_state(states: list[Any]) -> Any | None: + completed = [s for s in states if getattr(s, "group", None) == "completed"] + if not completed: + return None + # Prefer a non-default completed state named Done if present. + for s in completed: + if (s.name or "").strip().casefold() == "done": + return s + return completed[0] + + +def _seed_items(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + project_id = ctx["project_id"] + states = _list_states(plane, workspace_slug, project_id) + ctx["state_names"] = sorted({(s.name or "").strip() for s in states if (s.name or "").strip()}) + + # Prefer a non-default started-group state so R1 cannot be passed by guessing the default. + started = [s for s in states if getattr(s, "group", None) == "started" and not getattr(s, "default", False)] + if not started: + started = [s for s in states if getattr(s, "group", None) == "started"] + if not started: + raise RuntimeError( + "seed items: no started-group state available to place the R1 target; " + f"states={[(s.name, s.group, s.default) for s in states]}" + ) + r1_state = started[0] + ctx["r1_state_name"] = r1_state.name + ctx["r1_state_id"] = r1_state.id + + me = plane.users.get_me() + me_id = str(me.id) + ctx["me_id"] = me_id + # Due dates must stay inside the current ISO week (Mon–Sun). + # today+2d alone escapes the week on Sat/Sun — clamp to this week's Sunday. + today = date.today() + days_to_week_end = 6 - today.weekday() # Mon=0 … Sun=6 + due_this_week = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)).isoformat() + ctx["r3_due_date"] = due_this_week + + urgent_count = 0 + for title, priority in ITEM_FIXTURES: + data_kwargs: dict[str, Any] = {"name": title, "priority": priority} + if title == R1_TITLE: + data_kwargs["state"] = str(r1_state.id) + if title in R3_DUE_TITLES: + data_kwargs["assignees"] = [me_id] + data_kwargs["target_date"] = due_this_week + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(**data_kwargs), # type: ignore[arg-type] + ) + # Some APIs ignore state on create; force via update if needed. + if title == R1_TITLE: + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(r1_state.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(r1_state.id)), + ) + ctx["items"][title] = item.id + ctx["item_ids"].append(item.id) + seq = getattr(item, "sequence_id", None) + if seq is not None and ctx.get("project_identifier"): + ctx["item_identifiers"][title] = f"{ctx['project_identifier']}-{seq}" + if priority == "urgent": + urgent_count += 1 + assert urgent_count == 4, f"fixture invariant: expected 4 urgent items, got {urgent_count}" + + # R5: seed discussion comments on the known item. + r5_id = ctx["items"].get(R5_TITLE) + if r5_id: + for phrase in R5_COMMENT_PHRASES: + plane.work_items.comments.create( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=r5_id, + data=CreateWorkItemComment(comment_html=f"

{phrase}

"), + ) + + +def _gate_activity_worker(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + """Skip L2 when comments never materialize as activities (no activity worker). + + Raises :class:`evals.tasks.TaskSkipped` with reason ``env:no-activity-worker`` + so the harness records a skip, not a task failure. + """ + from evals.tasks import TaskSkipped + + project_id = ctx.get("project_id") + wid = (ctx.get("items") or {}).get(R5_TITLE) + if not project_id or not wid: + raise TaskSkipped("env:no-activity-worker") + try: + page = plane.work_items.activities.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=wid, + ) + except Exception as exc: + raise TaskSkipped(f"env:no-activity-worker ({type(exc).__name__}: {exc})") from exc + rows = page.results if hasattr(page, "results") else page + if len(list(rows or [])) < 1: + raise TaskSkipped("env:no-activity-worker") + + +def _seed_bug_type(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + """Create or resolve a 'Bug' work item type. + + Genuine plan-gate responses set bug_type=None + skip reason; all other failures raise. + Workspace feature probe uses the real key `is_work_item_types_enabled` (F10). + """ + project_id = ctx["project_id"] + target = "Bug" + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional work_item_types key alone. + workspace_owns = bool(dump.get("is_work_item_types_enabled")) + + if workspace_owns: + existing = next( + ( + t + for t in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) + if (t.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.workspace_work_item_types.create( + workspace_slug=workspace_slug, data=CreateWorkItemType(name=target) + ) + created = True + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_ids=[existing.id], + ) + ctx["bug_type"] = {"id": existing.id, "name": target} + ctx["bug_type_created"] = created + ctx["bug_type_workspace_level"] = True + if created: + ctx["workspace_objects"].append({"kind": "work_item_type", "id": existing.id}) + return + + # Per-project types. Project features expose no work-item-type toggle — do not PATCH. + existing = next( + ( + t + for t in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) + if (t.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.work_item_types.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItemType(name=target), + ) + created = True + ctx["bug_type"] = {"id": existing.id, "name": target} + ctx["bug_type_created"] = created + ctx["bug_type_workspace_level"] = False + except Exception as exc: + if _is_plan_gate(exc): + ctx["bug_type"] = None + ctx["bug_type_skip_reason"] = f"bug_type plan-gated: {exc}" + return + raise + + +def _seed_cycles(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any], leave_past_open: bool = False) -> None: + """Seed Sprint 12 (past) + Sprint 13 (active) with work items. + + Plane forbids adding issues to a cycle whose end_date is already past + (``The Cycle has already been completed so no new issues can be added`` — + plane-ee cycle/issue.py). Ordering for Sprint 12: + + 1. create with an *active* window (start past, end future) + 2. add_work_items while still active + 3. update end_date to the past (backdate) so the cycle is completed + + ``leave_past_open`` skips step 3, leaving Sprint 12 ending tomorrow. Closing a + cycle is only legal while it is still open — Plane rejects every edit to an + ended cycle (``The Cycle has already been completed so it cannot be edited``) + and rejects a transfer out of a still-running one (``The old cycle is not + completed yet``), so a fixture that pre-closes Sprint 12 makes "close it" + unachievable and leaves ``progress_snapshot`` (a transfer side effect) as the + only observable close signal. W6 asks the agent to close, so it seeds open. + + Sprint 13 is created and populated while genuinely active (start ≤ today ≤ end). + """ + project_id = ctx["project_id"] + me_id = ctx.get("me_id") or str(plane.users.get_me().id) + today = date.today() + # Final past window for Sprint 12 after backdate (completedCycles / W6 transfer source). + past_start = (today - timedelta(days=28)).isoformat() + past_end_final = (today - timedelta(days=14)).isoformat() + # Temporary active end so create + add succeed (end must be ≥ now). When the + # cycle stays open this is its final window, so keep it short — Sprint 12 ends + # tomorrow, which is what makes "close it and roll the rest over" natural. + past_end_active = (today + timedelta(days=1 if leave_past_open else 7)).isoformat() + # Sprint 13: genuinely active at seed time (start ≤ today ≤ end). + cur_start = (today - timedelta(days=3)).isoformat() + cur_end = (today + timedelta(days=10)).isoformat() + + # 1) Create Sprint 12 still active (items can be added). + past = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=CYCLE_PAST, + start_date=past_start, + end_date=past_end_active, + owned_by=me_id, + project_id=str(project_id), + ), + ) + # Sprint 13: active window for R4 / W6 transfer target. + current = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=CYCLE_CURRENT, + start_date=cur_start, + end_date=cur_end, + owned_by=me_id, + project_id=str(project_id), + ), + ) + ctx["cycles"] = { + CYCLE_PAST: past.id, + CYCLE_CURRENT: current.id, + } + ctx["cycle_past_id"] = past.id + ctx["cycle_current_id"] = current.id + + # 2) Add unfinished items to Sprint 12 *before* backdating. + unfinished_ids = [ctx["items"][t] for t in W6_UNFINISHED_TITLES if t in ctx["items"]] + if unfinished_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + issue_ids=unfinished_ids, + ) + # R4: items on the active cycle (window still open). + active_ids: list[str] = [] + for title in (R1_TITLE, "Session cookie not rotated after login"): + iid = ctx["items"].get(title) + if iid: + active_ids.append(iid) + if active_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + issue_ids=active_ids, + ) + overdue_id = ctx["items"].get("Session cookie not rotated after login") + if overdue_id: + plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=overdue_id, + data=UpdateWorkItem(target_date=(today - timedelta(days=3)).isoformat()), + ) + ctx["r4_overdue_title"] = "Session cookie not rotated after login" + ctx["r4_overdue_id"] = overdue_id + ctx["r4_active_item_ids"] = active_ids + + # 3) Backdate Sprint 12 so it is a completed cycle for R4 semantics — unless the + # task needs to close it itself, in which case it must still be open. + # UpdateCycle.end_date is writable; API allows past end_dates (no "can't backdate" gate + # on the update path — only add_work_items checks end_date < now). + if not leave_past_open: + plane.cycles.update( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + data=UpdateCycle(end_date=past_end_final), + ) + # Final seeded end_date for W6 close assertion (complete_cycle sets end_date=today). + ctx["cycle_past_seed_end_date"] = past_end_active if leave_past_open else past_end_final + ctx["cycle_past_open"] = leave_past_open + ctx["cycle_past_end_date_before_backdate"] = past_end_active + + +def _seed_module(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + project_id = ctx["project_id"] + states = _list_states(plane, workspace_slug, project_id) + done = _completed_state(states) + if done is None: + raise RuntimeError("seed module: no completed-group state to place module items") + + mod = plane.modules.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateModule(name=MODULE_NAME, status="in-progress"), + ) + ctx["module"] = {"id": mod.id, "name": MODULE_NAME} + completed_ids: list[str] = [] + for title in MODULE_COMPLETED_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(name=title, priority="medium", state=str(done.id)), # type: ignore[arg-type] + ) + # Force completed state if create ignored it. + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(done.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(done.id)), + ) + completed_ids.append(item.id) + ctx["items"][title] = item.id + ctx["item_ids"].append(item.id) + plane.modules.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + module_id=mod.id, + issue_ids=completed_ids, + ) + ctx["module_completed_ids"] = completed_ids + ctx["module_completed_state_id"] = done.id + + +def _seed_intake(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + project_id = ctx["project_id"] + billing = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_BILLING_TITLE, priority="high"), + ), + ) + spam = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_SPAM_TITLE, priority="none"), + ), + ) + # IntakeWorkItem.issue is the work-item id used by triage tools. + ctx["intake"] = { + "billing": { + "intake_id": billing.id, + "issue_id": getattr(billing, "issue", None) or billing.id, + "title": INTAKE_BILLING_TITLE, + }, + "spam": { + "intake_id": spam.id, + "issue_id": getattr(spam, "issue", None) or spam.id, + "title": INTAKE_SPAM_TITLE, + }, + } + + +def _seed_customer(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + customer = plane.customers.create( + workspace_slug=workspace_slug, + data=CreateCustomer(name=CUSTOMER_NAME), + ) + ctx["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} + ctx["workspace_objects"].append({"kind": "customer", "id": customer.id}) + req = plane.customers.requests.create( + workspace_slug=workspace_slug, + customer_id=customer.id, + data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), + ) + ctx["customer_request"] = {"id": req.id, "name": CUSTOMER_REQUEST_NAME, "customer_id": customer.id} + + +def _seed_release(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + rel = plane.releases.create( + workspace_slug=workspace_slug, + data=CreateRelease(name=RELEASE_NAME), + ) + ctx["release"] = {"id": rel.id, "name": RELEASE_NAME} + ctx["workspace_objects"].append({"kind": "release", "id": rel.id}) + # Single changelog body; DESIGN's "2 entries" are encoded as plain-text bullets. + try: + plane.releases.changelog.update( + workspace_slug=workspace_slug, + release_id=rel.id, + data=UpdateReleaseChangelog( + description_html=f"

{RELEASE_CHANGELOG_TEXT}

", + ), + ) + except Exception as exc: + # Non-fatal for seed if changelog endpoint is flaky; C2 verifier still checks release name. + print(f"seed warning: release changelog update failed: {exc}") + ctx["release_changelog_text"] = RELEASE_CHANGELOG_TEXT + + +def _seed_second_project(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: + """R6: second project with more open Bug items than the main eval project.""" + run8 = ctx["run8"] + name = f"EVAL {run8} B" + project = create_project_with_identifier_retry( + plane, + workspace_slug, + name=name, + identifier_prefix="EB", + initial_suffix=run8[:4].upper(), + ) + ctx["second_project_id"] = project.id + ctx["second_project_name"] = name + ctx["second_project_identifier"] = getattr(project, "identifier", None) + # Track for teardown (project delete covers it; still record). + ctx["second_project_ids"] = [project.id] + _enable_project_features(plane, workspace_slug, project.id) + + # Ensure Bug type exists on both projects. + if not ctx.get("bug_type"): + _seed_bug_type(plane, workspace_slug, ctx) + bug = ctx.get("bug_type") or {} + bug_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_id: + raise RuntimeError("seed second_project: bug_type required for R6 bug counts") + + # Import workspace-level type into second project when needed. + if ctx.get("bug_type_workspace_level"): + try: + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project.id, + work_item_type_ids=[bug_id], + ) + except Exception as exc: + if not _is_plan_gate(exc): + # May already be imported. + if not (isinstance(exc, HttpError) and exc.status_code in (400, 409)): + raise + + main_id = ctx["project_id"] + # Main project: fewer bugs + main_bug_ids: list[str] = [] + for title in R6_MAIN_BUG_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=main_id, + data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + ) + main_bug_ids.append(item.id) + ctx["items"][title] = item.id + ctx["item_ids"].append(item.id) + # Second project: more bugs + second_bug_ids: list[str] = [] + for title in R6_SECOND_BUG_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project.id, + data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + ) + second_bug_ids.append(item.id) + ctx["r6_main_bug_count"] = len(main_bug_ids) + ctx["r6_second_bug_count"] = len(second_bug_ids) + ctx["r6_more_bugs_project"] = name # second project has more + + +def _cleanup_severity_on_bug_type(plane: PlaneClient, ctx: dict[str, Any]) -> None: + """Delete Severity properties attached to the seeded Bug type (avoids multi-rep pollution).""" + bug = ctx.get("bug_type") + if not bug: + return + bug_type_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_type_id: + return + workspace_slug = ctx.get("workspace_slug") or "" + project_id = ctx.get("project_id") + + props: list[Any] = [] + try: + if project_id: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + except HttpError as exc: + if exc.status_code not in (404, 405): + print(f"teardown warning: list Severity props failed: {exc}") + return + except Exception as exc: + print(f"teardown warning: list Severity props failed: {exc}") + return + + for p in props: + display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() + if display.lower() != "severity": + continue + try: + if project_id: + plane.work_item_properties.delete( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + work_item_property_id=p.id, + ) + ctx.setdefault("workspace_objects", []) # no-op anchor + except Exception as exc: + print(f"teardown warning: failed to delete Severity property {p.id}: {exc}") + + +def _cleanup_agent_incident_type(plane: PlaneClient, ctx: dict[str, Any]) -> None: + """Best-effort cleanup of agent-created Incident type (S3 multi-rep pollution).""" + workspace_slug = ctx.get("workspace_slug") or "" + project_id = ctx.get("project_id") + try: + if ctx.get("bug_type_workspace_level"): + for t in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []: + if (t.name or "").strip().casefold() == "incident": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=t.id) + elif project_id: + for t in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []: + if (t.name or "").strip().casefold() == "incident": + plane.work_item_types.delete( + workspace_slug=workspace_slug, project_id=project_id, work_item_type_id=t.id + ) + except Exception as exc: + print(f"teardown warning: Incident type cleanup failed: {exc}") + + +def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: + """Delete the project and any workspace-scoped objects we created.""" + if not ctx: + return + workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + project_id = ctx.get("project_id") + + # S5 left customers off (or agent enabled them): re-enable for subsequent task-reps + # on the shared eval workspace. We always set customers=True — we do not restore a + # prior false (S5's job is to leave the workspace usable for C1). + if ctx.get("s5_left_customers_off"): + try: + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(customers=True), + ) + except Exception as exc: + print(f"teardown warning: re-enable workspace customers failed: {exc}") + + # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). + try: + _cleanup_severity_on_bug_type(plane, ctx) + except Exception as exc: + print(f"teardown warning: Severity cleanup failed: {exc}") + try: + _cleanup_agent_incident_type(plane, ctx) + except Exception as exc: + print(f"teardown warning: Incident cleanup failed: {exc}") + + # Best-effort: agent-created Acme Corp customers (C1) that never hit workspace_objects. + try: + page = plane.customers.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + for c in rows or []: + if (c.name or "").strip().casefold() in (CUSTOMER_NAME.casefold(), "acme"): + # Only delete if we seeded or created during this run (tracked or name match + run). + tracked = {o.get("id") for o in (ctx.get("workspace_objects") or []) if o.get("kind") == "customer"} + if str(c.id) in tracked or ctx.get("customer") is None: + # Avoid deleting long-lived Acme if we pre-seeded and tracked it — still delete tracked. + if str(c.id) in tracked or not ctx.get("customer"): + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": c.id}) + except Exception as exc: + print(f"teardown warning: customer scan failed: {exc}") + + # Workspace-scoped cleanup first (survive project deletion). + seen_ws: set[str] = set() + for obj in ctx.get("workspace_objects") or []: + kind = obj.get("kind") + oid = obj.get("id") + if not oid: + continue + key = f"{kind}:{oid}" + if key in seen_ws: + continue + seen_ws.add(key) + try: + if kind == "work_item_type": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=oid) + elif kind == "work_item_property": + plane.workspace_work_item_properties.delete(workspace_slug=workspace_slug, property_id=oid) + elif kind == "customer": + plane.customers.delete(workspace_slug=workspace_slug, customer_id=oid) + elif kind == "release": + plane.releases.delete(workspace_slug=workspace_slug, release_id=oid) + elif kind == "release_tag": + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=oid) + elif kind == "customer_property": + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=oid) + except Exception as exc: + print(f"teardown warning: failed to delete workspace {kind} {oid}: {exc}") + + # Sweep by well-known WS3 names in case tracking missed an agent-created row. + try: + page = plane.releases.tags.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + for tag in rows or []: + if (getattr(tag, "version", None) or "").strip() == DEBIAS_RELEASE_TAG_VERSION: + tid = getattr(tag, "id", None) + if tid and f"release_tag:{tid}" not in seen_ws: + try: + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tid) + except Exception as exc: + print(f"teardown warning: sweep release tag {tid}: {exc}") + except Exception as exc: + print(f"teardown warning: sweep release tags failed: {exc}") + try: + page = plane.customers.properties.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + target = DEBIAS_CUSTOMER_PROP_DISPLAY.casefold() + for prop in rows or []: + display = (getattr(prop, "display_name", None) or getattr(prop, "name", None) or "").strip() + if display.casefold() == target: + pid = getattr(prop, "id", None) + if pid and f"customer_property:{pid}" not in seen_ws: + try: + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=pid) + except Exception as exc: + print(f"teardown warning: sweep customer property {pid}: {exc}") + except Exception as exc: + print(f"teardown warning: sweep customer properties failed: {exc}") + + # Second project before main (no dependency either way, but be thorough). + for pid in ctx.get("second_project_ids") or []: + if not pid or pid == project_id: + continue + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=pid) + except Exception as exc: + print(f"teardown warning: failed to delete second project {pid}: {exc}") + + if project_id: + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + name = ctx.get("project_name", project_id) + print(f"teardown warning: failed to delete project {name!r}: {exc}") + print(f"orphaned project: {name}") diff --git a/evals/tasks.py b/evals/tasks.py new file mode 100644 index 00000000..43e5ad36 --- /dev/null +++ b/evals/tasks.py @@ -0,0 +1,2798 @@ +"""Task definitions (plain dicts) and verifier functions for the eval harness.""" + +from __future__ import annotations + +import hashlib +import json +import re +import string +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.enums import PropertyType +from plane.models.query_params import RetrieveQueryParams, WorkItemQueryParams + +from evals.seed import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + CYCLE_CURRENT, + CYCLE_PAST, + DEBIAS_CUSTOMER_PROP_DISPLAY, + DEBIAS_RELEASE_TAG_VERSION, + INTAKE_BILLING_TITLE, + INTAKE_SPAM_TITLE, + MODULE_COMPLETED_TITLES, + MODULE_NAME, + R1_TITLE, + R5_COMMENT_PHRASES, + R5_TITLE, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, + W2_TITLE, + W3_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, + W7_URL, + W8_TITLE, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class TaskSkipped(Exception): + """Verifier signals that this task-rep should be recorded as skipped, not failed.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class PromptBindError(RuntimeError): + """Live prompt could not bind required seed IDs (classified as infra_seed).""" + + +def format_task_prompt( + task: dict[str, Any], + ctx: dict[str, Any] | None = None, + *, + strict: bool = False, +) -> str: + """Render a task prompt with seed-bound placeholders. + + Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand + the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an + optional ``prompt_bind(ctx) -> dict`` callable on the task dict. + + When ``strict=True`` (live runs), empty-string values or binder exceptions + raise ``PromptBindError`` so the harness records ``infra_seed`` rather than + sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and + fills missing keys with explicit ```` markers. + """ + tpl = str(task.get("prompt") or "") + fields: dict[str, Any] = { + "project": (ctx or {}).get("project_name") or "EVAL deadbeef", + } + binder = task.get("prompt_bind") + if callable(binder) and ctx is not None: + try: + extra = binder(ctx) or {} + except Exception as exc: + if strict: + raise PromptBindError( + f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" + ) from exc + extra = {} + if isinstance(extra, dict): + for key, val in extra.items(): + if val is None: + if strict: + raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") + continue + text = str(val).strip() + if not text: + if strict: + raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") + continue + fields[key] = text + # Collect required placeholders from the template. + required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] + for name in required: + if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): + continue + if strict: + raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") + fields.setdefault(name, f"<{name}>") + return tpl.format(**fields) + + +def _word_boundary(value: str) -> re.Pattern[str]: + """Compile a case-insensitive word-boundary match for an exact seeded value.""" + return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) + + +def _reports_exact_int(text: str, n: int) -> bool: + """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" + return bool(_word_boundary(str(int(n))).search(text or "")) + + +def _whole_answer_int(text: str) -> int | None: + """If the answer (or its last non-empty line) is exactly an integer, return it. + + Letters must not appear — only surrounding whitespace/punctuation is ignored — + so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading + minus** attached to the number is preserved (``-3`` → -3, not 3). + """ + + def _as_int(s: str) -> int | None: + # Collapse whitespace; then the whole string must be optional sign + digits + # with only non-word punctuation wrappers (prefix must not eat the sign). + compact = re.sub(r"\s+", "", s or "") + m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) + if m: + return int(m.group(1)) + return None + + blob = text or "" + v = _as_int(blob) + if v is not None: + return v + lines = [ln for ln in blob.splitlines() if ln.strip()] + if lines: + return _as_int(lines[-1]) + return None + + +def reports_contract_int(text: str, truth: int) -> bool: + """True when final text reports ``truth`` via the explicit ``count: N`` contract. + + 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding + whitespace allowed). Use the **last** match; require signed equality with + ``truth``. + 2. Fallback: whole-answer / last-line bare integer (:func:`_whole_answer_int`). + 3. No match at all → False (ignoring an explicit format instruction is a fail). + """ + last: int | None = None + for line in (text or "").splitlines(): + m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) + if m: + last = int(m.group(1)) + if last is not None: + return last == int(truth) + whole = _whole_answer_int(text) + if whole is not None: + return whole == int(truth) + return False + + +def _as_id(obj: Any) -> str | None: + if obj is None: + return None + if isinstance(obj, str): + return obj + return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) + + +def _ids(items: Any) -> set[str]: + out: set[str] = set() + for item in items or []: + i = _as_id(item) + if i: + out.add(str(i)) + return out + + +def _find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: + """Return all work items with exact name, newest first (by created_at).""" + matches: list[Any] = [] + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in page.results or []: + if (item.name or "").strip() == name: + matches.append(item) + if not page.next_page_results: + break + cursor = page.next_cursor + + def _created_key(item: Any) -> str: + return str(getattr(item, "created_at", None) or "") + + matches.sort(key=_created_key, reverse=True) + return matches + + +def _find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: + """Locate a work item by exact name; when duplicates exist, prefer the newest.""" + matches = _find_items_by_name(plane, workspace_slug, project_id, name) + return matches[0] if matches else None + + +def _state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + """Resolve a state UUID or expanded object to its display name.""" + if state_ref is None: + return None + if hasattr(state_ref, "name") and state_ref.name: + return str(state_ref.name) + if isinstance(state_ref, dict) and state_ref.get("name"): + return str(state_ref["name"]) + state_id = _as_id(state_ref) + if not state_id: + return None + try: + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return state.name + except HttpError as exc: + if exc.status_code not in (404, 405): + raise + # Fall back to listing states and matching by id. + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + results = page.results if hasattr(page, "results") else page + for s in results or []: + if str(s.id) == str(state_id): + return s.name + return None + + +def _state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + if state_ref is None: + return None + if hasattr(state_ref, "group") and state_ref.group: + return str(state_ref.group) + if isinstance(state_ref, dict) and state_ref.get("group"): + return str(state_ref["group"]) + state_id = _as_id(state_ref) + if not state_id: + return None + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + for s in page.results or []: + if str(s.id) == str(state_id): + return getattr(s, "group", None) + return None + + +def _is_not_found(exc: BaseException) -> bool: + return isinstance(exc, HttpError) and exc.status_code in (404, 405) + + +def _final_text(run: dict[str, Any]) -> str: + return run.get("final_text") or "" + + +def _count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: + """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} + n = 0 + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in resp.results or []: + if (getattr(item, "priority", None) or "").lower() != "urgent": + continue + sid = _as_id(item.state) + if sid and str(sid) in closed_ids: + continue + n += 1 + if not resp.next_page_results: + break + cursor = resp.next_cursor + return n + + +# --------------------------------------------------------------------------- +# Verifiers — async (plane, ctx, run) -> (bool, note) +# --------------------------------------------------------------------------- + + +async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R1: final text must name the target item's state and no other seeded state. + + Matching rule: word-boundary, case-insensitive regex on the exact state name + resolved from the API at verify time (never hardcoded). Additionally fail if + any *other* project state name also matches (blocks guessing/list_states echo). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + title = R1_TITLE + item = _find_item_by_name(plane, workspace_slug, project_id, title) + if item is None: + return False, f"seeded item {title!r} not found" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + expected = _state_name(plane, workspace_slug, project_id, detail.state) + if not expected: + # Prefer the seeded name when API is sparse. + expected = ctx.get("r1_state_name") + if not expected: + return False, "could not resolve expected state name from API" + + final_text = _final_text(run) + if not _word_boundary(expected).search(final_text): + return False, f"final text missing state name {expected!r}" + + other_states = [n for n in (ctx.get("state_names") or []) if n and n.casefold() != expected.casefold()] + collisions = [n for n in other_states if _word_boundary(n).search(final_text)] + if collisions: + return ( + False, + f"final text names other state(s) {collisions!r} besides expected {expected!r}", + ) + return True, f"final text names only state {expected!r}" + + +async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R2: final text must contain the exact urgent-open count (word-boundary).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + expected = _count_open_urgent(plane, workspace_slug, project_id) + final_text = _final_text(run) + # Word-boundary on the decimal form of the count (blocks "4" matching "24"). + if not _word_boundary(str(expected)).search(final_text): + return False, f"final text missing urgent-open count {expected}" + return True, f"final text names count {expected}" + + +async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R3: final text must include each seeded assigned-to-me / due-this-week title.""" + titles = list(ctx.get("r3_due_titles") or []) + if not titles: + return False, "no R3 due titles in seed ctx" + final_text = _final_text(run) + missing = [t for t in titles if not _word_boundary(t).search(final_text)] + if missing: + return False, f"final text missing title(s) {missing!r}" + return True, f"final text names {len(titles)} due-this-week assigned items" + + +async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R4: final text must mention the active cycle name and the overdue item title.""" + final_text = _final_text(run) + notes: list[str] = [] + ok = True + if not _word_boundary(CYCLE_CURRENT).search(final_text): + ok = False + notes.append(f"missing active cycle {CYCLE_CURRENT!r}") + else: + notes.append(f"names {CYCLE_CURRENT}") + overdue = ctx.get("r4_overdue_title") + if overdue: + if not _word_boundary(overdue).search(final_text): + # Soft: also accept "overdue" keyword + any active item title. + if "overdue" not in final_text.casefold(): + ok = False + notes.append(f"missing overdue title {overdue!r}") + else: + notes.append("mentions overdue (title not exact)") + else: + notes.append(f"names overdue {overdue!r}") + return ok, "; ".join(notes) + + +async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R5: final text must include seeded comment phrases (word-boundary).""" + phrases = list(ctx.get("r5_comment_phrases") or R5_COMMENT_PHRASES) + final_text = _final_text(run) + missing = [p for p in phrases if not _word_boundary(p).search(final_text)] + if missing: + return False, f"final text missing comment phrase(s) {missing!r}" + return True, f"final text names {len(phrases)} discussion phrases" + + +async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R6: final text must name the project that has more open bugs (resolved at verify).""" + expected = ctx.get("r6_more_bugs_project") or ctx.get("second_project_name") + if not expected: + return False, "second project name missing from seed ctx" + final_text = _final_text(run) + # Match the full project name or the distinctive " B" suffix run8 form. + if _word_boundary(expected).search(final_text): + return True, f"final text names project with more bugs {expected!r}" + # Allow matching just the identifier-ish trailing token (e.g. run8 + B). + run8 = ctx.get("run8") or "" + alt = f"EVAL {run8} B" + if _word_boundary(alt).search(final_text) or (run8 and run8 in final_text and " B" in final_text): + return True, f"final text names second project ({alt})" + return False, f"final text missing project with more bugs {expected!r}" + + +async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W1: assert end-state via Plane API (title, priority, assignee, auth label).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + title = "Login page 500s on empty password" + matches = _find_items_by_name(plane, workspace_slug, project_id, title) + if not matches: + return False, f"work item {title!r} not found" + item = matches[0] # newest first + notes: list[str] = [] + if len(matches) > 1: + notes.append(f"warning: {len(matches)} items with title (verifying newest)") + + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + params=RetrieveQueryParams(expand="assignees,labels"), + ) + + ok = True + + priority = (detail.priority or "").lower() if detail.priority else "" + if priority != "urgent": + ok = False + notes.append(f"priority={priority!r} (want urgent)") + else: + notes.append("priority=urgent") + + me = plane.users.get_me() + me_id = str(me.id) + assignee_ids = _ids(detail.assignees) + if me_id not in assignee_ids: + ok = False + notes.append(f"assignees={sorted(assignee_ids)} missing me={me_id}") + else: + notes.append("assigned to me") + + auth_label_id = (ctx.get("labels") or {}).get("auth") + label_ids = _ids(detail.labels) + if not auth_label_id: + ok = False + notes.append("auth label id missing from seed ctx") + elif str(auth_label_id) not in label_ids: + ok = False + notes.append(f"labels={sorted(label_ids)} missing auth={auth_label_id}") + else: + notes.append("auth label attached") + + return ok, "; ".join(notes) + + +async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W2: target item is in a completed-group state (prefer name Done).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = _find_item_by_name(plane, workspace_slug, project_id, W2_TITLE) + if item is None: + return False, f"item {W2_TITLE!r} not found" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + name = _state_name(plane, workspace_slug, project_id, detail.state) + group = _state_group(plane, workspace_slug, project_id, detail.state) + if group == "completed" or (name and name.casefold() == "done"): + return True, f"state={name!r} group={group!r}" + return False, f"state={name!r} group={group!r} (want completed/Done)" + + +async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W3: target item has a comment containing the prompt phrase 'contrast tokens'.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = _find_item_by_name(plane, workspace_slug, project_id, W3_TITLE) + if item is None: + return False, f"item {W3_TITLE!r} not found" + resp = plane.work_items.comments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + results = list(resp.results if hasattr(resp, "results") else resp or []) + if not results: + return False, "no comments on target item" + phrase = "contrast tokens" + pat = _word_boundary(phrase) + for c in results: + html = getattr(c, "comment_html", None) or "" + stripped = getattr(c, "comment_stripped", None) or "" + # Some APIs expose plain text under comment_stripped; fall back to html. + blob = f"{stripped}\n{html}" + if pat.search(blob): + return True, f"comment matches {phrase!r}" + return False, f"no comment contains {phrase!r} ({len(results)} comment(s))" + + +async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W4: the seeded triage label id is now named needs-triage. + + Authoritative path: retrieve ctx['labels']['triage'] by id. Name-scan is + only a fallback when the seed id is missing from ctx. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + triage_id = (ctx.get("labels") or {}).get("triage") + if triage_id: + try: + lb = plane.labels.retrieve(workspace_slug=workspace_slug, project_id=project_id, label_id=triage_id) + name = (lb.name or "").strip().casefold() + if name in ("needs-triage", "needs triage"): + return True, f"label id {triage_id} now named {lb.name!r}" + return False, f"label id {triage_id} still named {lb.name!r}" + except HttpError as exc: + if not _is_not_found(exc): + raise + return False, f"seeded triage label id {triage_id} not found (deleted?)" + + # Fallback only when seed id is absent from ctx. + page = plane.labels.list(workspace_slug=workspace_slug, project_id=project_id) + names = {(lb.name or "").strip().casefold(): (lb.name or "").strip() for lb in (page.results or [])} + if "needs-triage" in names or "needs triage" in names: + if "triage" in names: + return False, "both triage and needs-triage still present" + return True, "label renamed to needs-triage (no seed id; name-scan fallback)" + return False, f"needs-triage not found; labels={sorted(names.values())}" + + +async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W5: all seeded module completed items are archived (not merely deleted).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + ids = [str(i) for i in (ctx.get("module_completed_ids") or [])] + if not ids: + # Fall back to titles. + for title in MODULE_COMPLETED_TITLES: + item = _find_item_by_name(plane, workspace_slug, project_id, title) + if item: + ids.append(str(item.id)) + if not ids: + return False, "no module completed item ids" + + not_archived: list[str] = [] + need_archive_list: list[str] = [] # 404 on retrieve — must appear in archived list + for wid in ids: + try: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except HttpError as exc: + if _is_not_found(exc): + # Deleted OR archived-as-404 — require confirmation via list_archived. + need_archive_list.append(str(wid)) + continue + raise + archived_at = getattr(detail, "archived_at", None) + if not archived_at: + not_archived.append(str(wid)) + + arch_ids: set[str] = set() + if need_archive_list or not_archived: + try: + arch = plane.work_items.list_archived( + workspace_slug=workspace_slug, + project_id=project_id, + params=WorkItemQueryParams(per_page=100), + ) + arch_ids = {str(i.id) for i in (arch.results or [])} + except Exception as exc: + if need_archive_list: + return False, f"list_archived failed while confirming 404 items: {exc}" + + # 404s only count as archived if present on the archived list (deletes fail). + for wid in need_archive_list: + if wid not in arch_ids: + not_archived.append(wid) + not_archived = [i for i in not_archived if i not in arch_ids] + + if not_archived: + return False, f"{len(not_archived)} module items not archived: {not_archived}" + return True, f"{len(ids)} module completed items archived" + + +async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W6: Sprint 12 closed by a real completion signal + unfinished items on Sprint 13. + + complete_cycle (SDK) sets end_date to *today* — a no-op agent leaves the seeded + past end_date unchanged, so requiring end_date==today (or archived_at set) is + non-vacuous. progress_snapshot non-null is also accepted when the API flips it. + """ + from datetime import date as _date + + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + past_id = ctx.get("cycle_past_id") or (ctx.get("cycles") or {}).get(CYCLE_PAST) + cur_id = ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) + if not past_id: + return False, "Sprint 12 id missing from seed" + notes: list[str] = [] + ok = True + past = plane.cycles.retrieve(workspace_slug=workspace_slug, project_id=project_id, cycle_id=past_id) + end = getattr(past, "end_date", None) + archived_at = getattr(past, "archived_at", None) + snapshot = getattr(past, "progress_snapshot", None) + today = _date.today().isoformat() + seed_end = ctx.get("cycle_past_seed_end_date") + + # Real close signals (any one suffices): + # 1) complete_cycle → end_date becomes today + # 2) manage_cycle_archive → archived_at set + # 3) progress_snapshot populated (Plane completion snapshot) + # end_date comes back as a timestamp ('2026-08-12T00:00:00Z'), so compare the + # date part — a whole-string match against today's date can never be true. + end_day = str(end or "")[:10] + closed = False + if archived_at: + closed = True + notes.append(f"Sprint 12 archived_at={archived_at}") + elif end_day == today: + closed = True + notes.append(f"Sprint 12 end_date={end} (complete_cycle today)") + elif snapshot not in (None, {}, []): + closed = True + notes.append("Sprint 12 progress_snapshot set") + if not closed: + ok = False + notes.append( + f"Sprint 12 not closed: end_date={end!r} seed_end={seed_end!r} " + f"archived_at={archived_at!r} snapshot={snapshot!r} " + f"(want end_date={today!r} or archived_at or progress_snapshot)" + ) + + unfinished = list(ctx.get("w6_unfinished_titles") or []) + if cur_id and unfinished: + try: + on13 = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=cur_id, + params=WorkItemQueryParams(per_page=100), + ) + names = {(i.name or "").strip() for i in (on13.results or [])} + missing = [t for t in unfinished if t not in names] + if missing: + ok = False + notes.append(f"unfinished not on Sprint 13: {missing}") + else: + notes.append(f"{len(unfinished)} unfinished on Sprint 13") + except Exception as exc: + notes.append(f"list Sprint 13 items failed: {exc}") + return ok, "; ".join(notes) + + +async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W7: source blocks target (dependency) AND reference URL link exists on source. + + Only dump['blocking'] ids count — a reverse blocked_by match must not pass. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + src = _find_item_by_name(plane, workspace_slug, project_id, W7_SOURCE_TITLE) + tgt = _find_item_by_name(plane, workspace_slug, project_id, W7_TARGET_TITLE) + if not src or not tgt: + return False, "W7 source/target items not found" + notes: list[str] = [] + ok = True + + # Dependencies — require tgt in blocking specifically. + try: + deps = plane.work_items.dependencies.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + dump = deps.model_dump() if hasattr(deps, "model_dump") else (deps if isinstance(deps, dict) else {}) + blocking = dump.get("blocking") or [] + if isinstance(blocking, dict): + blocking = blocking.get("results") or list(blocking.values()) + blocking_ids = _ids(blocking) + # blocking may also be plain UUID strings + for b in blocking if isinstance(blocking, list) else []: + if isinstance(b, str): + blocking_ids.add(b) + if str(tgt.id) not in blocking_ids: + ok = False + blob_hit = str(tgt.id) in str(dump) + note = f"no blocking relation from source to {tgt.id}; blocking_ids={sorted(blocking_ids)}" + if blob_hit: + note += " (target id appears elsewhere in dump — wrong direction)" + notes.append(note) + else: + notes.append("blocking relation present") + except Exception as exc: + ok = False + notes.append(f"dependencies list failed: {exc}") + + # Links + try: + links = plane.work_items.links.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + rows = links.results if hasattr(links, "results") else links + urls = {(getattr(ln, "url", None) or "").strip() for ln in (rows or [])} + if W7_URL not in urls: + ok = False + notes.append(f"link {W7_URL!r} missing; have {sorted(urls)}") + else: + notes.append("reference URL present") + except Exception as exc: + ok = False + notes.append(f"links list failed: {exc}") + + return ok, "; ".join(notes) + + +async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W8: work log of exactly 120 minutes exists on the target item. + + Note: plane-sdk Create work log has no logged-date field — 'yesterday' in the + prompt cannot be asserted; only duration is verified. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = _find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + return False, f"item {W8_TITLE!r} not found" + logs = plane.work_items.work_logs.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 120 in durations: + return True, "work log duration=120 present" + return False, f"no 120-minute work log; durations={durations}" + + +async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W9 (extra): bulk priority change — the three non-R1 urgent titles are now high.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # All urgent fixtures except we ask agent to set medium-priority batch targets. + # Prompt targets the three titles starting with Session/Inventory/Checkout (non-R1 urgent). + targets = [ + "Checkout times out on 3DS challenge", + "Session cookie not rotated after login", + "Inventory count goes negative under load", + ] + wrong: list[str] = [] + for title in targets: + item = _find_item_by_name(plane, workspace_slug, project_id, title) + if not item: + wrong.append(f"{title}: missing") + continue + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + pr = (detail.priority or "").lower() + if pr != "high": + wrong.append(f"{title}: priority={pr!r}") + if wrong: + return False, "; ".join(wrong) + return True, "3 items priority=high" + + +async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W10 (extra): project page named Eval Runbook exists.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + try: + resp = plane.pages.list_project_pages(workspace_slug=workspace_slug, project_id=project_id) + rows = resp.results if hasattr(resp, "results") else resp + except Exception as exc: + return False, f"list pages failed: {exc}" + names = {(getattr(p, "name", None) or "").strip() for p in (rows or [])} + if "Eval Runbook" not in names: + return False, f"page 'Eval Runbook' missing; have {sorted(names)}" + return True, "page Eval Runbook present" + + +async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S1: Bug type has an OPTION property 'Severity' with Critical/Major/Minor. + + Only type-scoped property listing is accepted (no project/workspace fallbacks that + would pass an unattached Severity). Unexpected API errors propagate as harness errors. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + bug_type_id = ( + (ctx.get("bug_type") or {}).get("id") if isinstance(ctx.get("bug_type"), dict) else ctx.get("bug_type") + ) + if not bug_type_id: + raise TaskSkipped("bug_type not seeded") + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + except HttpError as exc: + if _is_not_found(exc): + return False, "Severity property not found on Bug type (type-scoped list empty/404)" + raise + + severity = None + for p in props: + display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() + if display.lower() == "severity": + # Prefer an explicit type link when the API exposes it. + issue_type = getattr(p, "issue_type", None) + if issue_type is not None and str(issue_type) not in ("", str(bug_type_id)): + continue + severity = p + break + if severity is None: + return False, "Severity property not found on Bug type" + + prop_type = getattr(severity, "property_type", None) + prop_type_val = prop_type.value if isinstance(prop_type, PropertyType) else prop_type + if str(prop_type_val or "").upper() != PropertyType.OPTION.value: + return False, f"Severity property_type={prop_type_val!r} (want OPTION)" + + option_names = { + (getattr(o, "name", None) or (o.get("name") if isinstance(o, dict) else "") or "").strip() + for o in (getattr(severity, "options", None) or []) + } + if not option_names: + try: + opts = plane.work_item_properties.options.list( + workspace_slug=workspace_slug, + project_id=project_id, + property_id=severity.id, + ) + option_names = {(getattr(o, "name", None) or "").strip() for o in (opts or [])} + except HttpError as exc: + if not _is_not_found(exc): + raise + option_names = set() + + required = {"critical", "major", "minor"} + have = {n.casefold() for n in option_names if n} + missing = required - have + if missing: + return False, f"Severity options missing {sorted(missing)}; have {sorted(option_names)}" + return True, "Severity OPTION with Critical/Major/Minor present on Bug type" + + +async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S2: Fibonacci estimate scale exists and target item estimate_point is 5.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve active estimate + points. + try: + est = plane.estimates.retrieve(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + return False, f"no project estimate: {exc}" + est_id = getattr(est, "id", None) or _as_id(est) + points = plane.estimates.list_points(workspace_slug=workspace_slug, project_id=project_id, estimate_id=est_id) + point_rows = points if isinstance(points, list) else (points.results if hasattr(points, "results") else points) + values = {(getattr(p, "value", None) or "").strip() for p in (point_rows or [])} + fib_like = {"1", "2", "3", "5", "8"} + if not fib_like.issubset(values): + ok = False + notes.append(f"estimate points missing fib subset; have {sorted(values)}") + else: + notes.append("fibonacci points present") + + five = next((p for p in (point_rows or []) if (getattr(p, "value", None) or "").strip() == "5"), None) + item = _find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + ok = False + notes.append(f"target item {W8_TITLE!r} missing") + else: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + ep = getattr(detail, "estimate_point", None) + ep_id = _as_id(ep) if not isinstance(ep, (int, float)) else None + # estimate_point may be expanded object or UUID. + if five is not None and ep_id and str(ep_id) == str(five.id): + notes.append("item estimate_point=5") + elif ep is not None and str(getattr(ep, "value", ep)) in ("5", "5.0"): + notes.append("item estimate value=5") + else: + ok = False + notes.append(f"item estimate_point={ep!r} (want 5)") + return ok, "; ".join(notes) + + +async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S3: Incident type exists with a required TEXT property. + + Workspace-owned types: probe get_features.is_work_item_types_enabled at verify + time (S3 needs is empty, so seed never sets bug_type_workspace_level). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # Find Incident type (project list first). + types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) + incident = next((t for t in types if (t.name or "").strip().casefold() == "incident"), None) + if incident is None: + # Probe workspace feature at verify time — do not rely on seed ctx flags. + workspace_owns = False + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + workspace_owns = bool(dump.get("is_work_item_types_enabled")) + except Exception: + workspace_owns = False + if workspace_owns: + try: + wtypes = list(plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []) + incident = next((t for t in wtypes if (t.name or "").strip().casefold() == "incident"), None) + except Exception: + pass + if incident is None: + return False, "Incident work item type not found" + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(incident.id), + ) + or [] + ) + except HttpError as exc: + if _is_not_found(exc): + return False, "no properties on Incident type" + raise + + required_text = None + for p in props: + prop_type = getattr(p, "property_type", None) + prop_type_val = str(prop_type.value if isinstance(prop_type, PropertyType) else prop_type or "").upper() + is_required = bool(getattr(p, "is_required", False)) + # TEXT only — no fallback to other required types (OPTION etc.). + if is_required and prop_type_val in (PropertyType.TEXT.value, "TEXT", "STRING"): + required_text = p + break + if required_text is None: + return False, f"no required TEXT property on Incident; props={len(props)}" + display = getattr(required_text, "display_name", None) or getattr(required_text, "name", None) + return True, f"Incident type + required TEXT property {display!r}" + + +async def verify_s4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S4: billing intake accepted (status=1), spam declined (status=-1).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + intake = ctx.get("intake") or {} + billing = intake.get("billing") or {} + spam = intake.get("spam") or {} + notes: list[str] = [] + ok = True + + def _status_of(issue_id: str | None, title: str) -> int | None: + if not issue_id: + return None + try: + row = plane.intake.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=issue_id) + return getattr(row, "status", None) + except Exception: + # Fall back to list + match title + try: + rows = plane.intake.list(workspace_slug=workspace_slug, project_id=project_id) + results = rows.results if hasattr(rows, "results") else rows + for r in results or []: + detail = getattr(r, "issue_detail", None) + name = getattr(detail, "name", None) if detail is not None else None + if name and name.strip() == title: + return getattr(r, "status", None) + except Exception: + return None + return None + + b_status = _status_of(billing.get("issue_id"), INTAKE_BILLING_TITLE) + s_status = _status_of(spam.get("issue_id"), INTAKE_SPAM_TITLE) + # accept=1, decline=-1 per IntakeWorkItemStatusEnum + if b_status != 1: + ok = False + notes.append(f"billing status={b_status!r} (want 1/accepted)") + else: + notes.append("billing accepted") + if s_status != -1: + ok = False + notes.append(f"spam status={s_status!r} (want -1/declined)") + else: + notes.append("spam declined") + return ok, "; ".join(notes) + + +async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S5: project cycles + worklogs AND workspace customers enabled. + + Gates (plane-ee): + - project.cycle_view — cycles create/list + - project.is_time_tracking_enabled — worklogs + - WorkspaceFeature.is_customer_enabled (API field ``customers``) — customer create 403 + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + proj = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) + cycle_view = bool(getattr(proj, "cycle_view", None)) + time_tracking = bool(getattr(proj, "is_time_tracking_enabled", None)) + if not cycle_view: + ok = False + notes.append(f"cycle_view={getattr(proj, 'cycle_view', None)!r} (want True)") + else: + notes.append("cycle_view=True") + if not time_tracking: + ok = False + notes.append(f"is_time_tracking_enabled={getattr(proj, 'is_time_tracking_enabled', None)!r} (want True)") + else: + notes.append("is_time_tracking_enabled=True") + + try: + feat = plane.projects.get_features(workspace_slug=workspace_slug, project_id=project_id) + dump = feat.model_dump() if hasattr(feat, "model_dump") else (feat if isinstance(feat, dict) else {}) + cycles_flag = dump.get("cycles") if isinstance(dump, dict) else getattr(feat, "cycles", None) + if not cycles_flag: + ok = False + notes.append(f"features.cycles={cycles_flag!r} (want True)") + else: + notes.append("features.cycles=True") + except Exception as exc: + ok = False + notes.append(f"project get_features failed: {exc}") + + # Workspace customers toggle (is_customer_enabled behind API field ``customers``). + try: + ws_feat = plane.workspaces.get_features(workspace_slug=workspace_slug) + ws_dump = ( + ws_feat.model_dump() if hasattr(ws_feat, "model_dump") else (ws_feat if isinstance(ws_feat, dict) else {}) + ) + customers_on = None + if isinstance(ws_dump, dict): + customers_on = ws_dump.get("customers") + if customers_on is None: + customers_on = ws_dump.get("is_customer_enabled") + if customers_on is None: + customers_on = getattr(ws_feat, "customers", None) + if customers_on is None: + customers_on = getattr(ws_feat, "is_customer_enabled", None) + if not customers_on: + ok = False + notes.append(f"workspace.customers={customers_on!r} (want True)") + else: + notes.append("workspace.customers=True") + except Exception as exc: + ok = False + notes.append(f"workspace get_features failed: {exc}") + + return ok, "; ".join(notes) + + +async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C1: customer 'Acme Corp' has request 'SSO support' linked to the R1 work item. + + Anchors: exact customer name, exact request name, and the R1_TITLE work item id + resolved from the eval project at verify time (must be among linked ids). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve the required link target first (seeded Payment webhook item). + r1 = _find_item_by_name(plane, workspace_slug, project_id, R1_TITLE) + if r1 is None: + return False, f"R1 item {R1_TITLE!r} not found in project" + + customers = plane.customers.list(workspace_slug=workspace_slug) + rows = customers.results if hasattr(customers, "results") else customers + # Exact name only — do not match arbitrary acme* customers. + acme = next((c for c in (rows or []) if (c.name or "").strip() == CUSTOMER_NAME), None) + if acme is None: + return False, f"customer {CUSTOMER_NAME!r} not found" + + # Track for teardown if agent-created + if not ctx.get("customer"): + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": acme.id}) + + reqs = plane.customers.requests.list(workspace_slug=workspace_slug, customer_id=acme.id) + rrows = reqs.results if hasattr(reqs, "results") else reqs + sso = next( + (r for r in (rrows or []) if (r.name or "").strip() == CUSTOMER_REQUEST_NAME), + None, + ) + if sso is None: + ok = False + notes.append(f"request {CUSTOMER_REQUEST_NAME!r} missing") + else: + notes.append("SSO request present") + + # Require the R1 work item among customer-linked work items. + try: + wi = plane.customers.work_items.list(workspace_slug=workspace_slug, customer_id=acme.id) + wi_rows = list(wi.results if hasattr(wi, "results") else wi or []) + linked_ids = _ids(wi_rows) + # Plain string ids also count. + for row in wi_rows: + if isinstance(row, str): + linked_ids.add(row) + elif isinstance(row, dict) and row.get("id"): + linked_ids.add(str(row["id"])) + else: + # Customer work item wrappers may expose work_item / issue field. + for attr in ("work_item", "work_item_id", "issue", "issue_id"): + ref = getattr(row, attr, None) if not isinstance(row, dict) else row.get(attr) + rid = _as_id(ref) + if rid: + linked_ids.add(str(rid)) + if str(r1.id) not in linked_ids: + ok = False + notes.append(f"R1 item {r1.id} not linked; linked={sorted(linked_ids)}") + else: + notes.append(f"R1 item {r1.id} linked") + except Exception as exc: + ok = False + notes.append(f"list customer work items failed: {exc}") + + return ok, "; ".join(notes) + + +async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C2: final text mentions release 1.2.0 and at least one seeded changelog phrase.""" + final_text = _final_text(run) + notes: list[str] = [] + ok = True + if not _word_boundary(RELEASE_NAME).search(final_text): + ok = False + notes.append(f"missing release name {RELEASE_NAME!r}") + else: + notes.append(f"names {RELEASE_NAME}") + changelog = ctx.get("release_changelog_text") or RELEASE_CHANGELOG_TEXT + # Match distinctive fragments from the seeded changelog. + fragments = ["OAuth login hardening", "webhook retry backoff"] + hit = [f for f in fragments if _word_boundary(f).search(final_text)] + if not hit: + # Also accept substring of full changelog without word-boundary if short. + if changelog[:40].casefold() not in final_text.casefold(): + ok = False + notes.append("missing changelog content") + else: + notes.append("changelog substring present") + else: + notes.append(f"changelog phrases {hit}") + return ok, "; ".join(notes) + + +async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R7 (extra): final text names at least one legal next state for the R1 item. + + Resolves available completed/started/unstarted states at verify time and + requires a word-boundary hit on one of them (or explicit 'unrestricted'). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + names = [(s.name or "").strip() for s in (page.results or []) if (s.name or "").strip()] + final_text = _final_text(run) + if "unrestricted" in final_text.casefold() or "any state" in final_text.casefold(): + return True, "agent reported unrestricted transitions" + hits = [n for n in names if _word_boundary(n).search(final_text)] + if not hits: + return False, f"final text names none of project states {names}" + return True, f"final text names state(s) {hits}" + + +# --------------------------------------------------------------------------- +# ID-in-hand (I*) + long-tail (L*) de-biasing verifiers +# --------------------------------------------------------------------------- + +# Stable titles used by ID-in-hand binders (seeded by needs={"items", ...}). +I1_TITLE = R1_TITLE # update priority by UUID +I2_TITLE = W2_TITLE # fetch by PROJ-N identifier +I3_TITLE = "Footer year still says 2024" # add to cycle by UUIDs (not on a cycle) +I4_TITLE = W3_TITLE # attach label by UUIDs +L1_TITLE = W8_TITLE # worklog + project summary +L2_TITLE = R5_TITLE # activities (seeded comments produce activity rows) +L5_TITLE = R1_TITLE # attachment listing +L3_TAG_VERSION = DEBIAS_RELEASE_TAG_VERSION +L4_PROP_DISPLAY = DEBIAS_CUSTOMER_PROP_DISPLAY +L4_PROP_VALUE = "Enterprise" + + +def _bind_item_uuid(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(title) or "") + return {"work_item_id": wid} + + return _bind + + +def _bind_item_identifier(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + ident = str((ctx.get("item_identifiers") or {}).get(title) or "") + return {"work_item_identifier": ident} + + return _bind + + +def _bind_i3(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + return {"work_item_id": wid, "cycle_id": cycle_id} + + +def _bind_i4(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I4_TITLE) or "") + label_id = str((ctx.get("labels") or {}).get("perf") or "") + return {"work_item_id": wid, "label_id": label_id} + + +async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I1: seeded R1 item priority is high (updated by UUID, not name).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I1_TITLE) + if not wid: + return False, f"seed item {I1_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "high": + return True, f"work_item {wid} priority=high" + return False, f"work_item {wid} priority={pr!r} (want high)" + + +async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I2: final text names the state of the identifier-target item.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I2_TITLE) + if not wid: + return False, f"seed item {I2_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + name = _state_name(plane, workspace_slug, project_id, detail.state) + if not name: + return False, "target state name unresolved" + final_text = _final_text(run) + if _word_boundary(name).search(final_text): + return True, f"final text names state {name!r}" + return False, f"final text missing state {name!r}" + + +async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I3: work item UUID is on the target cycle UUID.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + if not wid or not cycle_id: + return False, "seed work_item_id/cycle_id missing" + page = plane.cycles.list_work_items(workspace_slug=workspace_slug, project_id=project_id, cycle_id=cycle_id) + rows = page.results if hasattr(page, "results") else page + ids = {str(getattr(r, "id", None) or r) for r in (rows or [])} + # list may return issue wrappers with issue/id fields + for r in rows or []: + for attr in ("id", "issue", "work_item_id"): + v = getattr(r, attr, None) + if v is not None: + ids.add(str(v if not hasattr(v, "id") else v.id)) + if wid in ids: + return True, f"item {wid} on cycle {cycle_id}" + return False, f"item {wid} not on cycle {cycle_id}; have {sorted(ids)[:12]}" + + +async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I4: work item has the seeded perf label id attached.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I4_TITLE) + label_id = (ctx.get("labels") or {}).get("perf") + if not wid or not label_id: + return False, "seed work_item_id/label_id missing" + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=wid, + params=RetrieveQueryParams(expand="labels"), + ) + label_ids = _ids(detail.labels) + if str(label_id) in label_ids: + return True, f"label {label_id} on {wid}" + return False, f"labels={sorted(label_ids)} missing perf={label_id}" + + +async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I5: target item priority is low (updated by UUID).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I3_TITLE) + if not wid: + return False, f"seed item {I3_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "low": + return True, f"work_item {wid} priority=low" + return False, f"work_item {wid} priority={pr!r} (want low)" + + +def _l1_duration_reported(final_text: str) -> bool: + """Numeric duration only: whole-word 90 or 1.5 (not English 'ninety').""" + return bool(_word_boundary("90").search(final_text)) or bool(re.search(r"\b1\.5\b", final_text)) + + +def _l1_person_names_from_summary(sum_rows: Any) -> list[str]: + """Best-effort actor/assignee display strings from project worklog summary rows.""" + names: list[str] = [] + for row in sum_rows or []: + dump = row.model_dump() if hasattr(row, "model_dump") else {} + if not isinstance(dump, dict): + dump = {} + candidates: list[Any] = [] + for attr in ( + "actor", + "user", + "display_name", + "owned_by", + "created_by", + "assignee", + "email", + "first_name", + "last_name", + ): + v = getattr(row, attr, None) + if v is None and dump: + v = dump.get(attr) + if v is None: + continue + if hasattr(v, "display_name") or hasattr(v, "email"): + candidates.append( + getattr(v, "display_name", None) or getattr(v, "email", None) or getattr(v, "id", None) + ) + elif isinstance(v, dict): + candidates.append(v.get("display_name") or v.get("email") or v.get("id")) + else: + candidates.append(v) + for c in candidates: + s = str(c or "").strip() + if s and s not in names: + names.append(s) + return names + + +def _l1_summary_substance(final_text: str, *, title: str, sum_rows: Any) -> bool: + """Summary half of L1: item title, person from summary, or words summary/total. + + Deliberately does *not* accept bare 'logged' / 'worklog' — the prompt asks to + report the project worklog summary (who/what has time logged). + """ + low = final_text.casefold() + if "summary" in low or "total" in low: + return True + if title and _word_boundary(title).search(final_text): + return True + for person in _l1_person_names_from_summary(sum_rows): + if len(person) >= 2 and _word_boundary(person).search(final_text): + return True + return False + + +async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L1: 90-minute work log on the correct item AND final text reports duration + summary. + + Duration: numeric whole-word ``90`` or ``1.5`` only (not English 'ninety'). + Summary substance: item title, a person/assignee from the project summary, or + the words ``summary`` / ``total``. Bare "90 minutes of work" fails by design. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L1_TITLE) + if not wid: + return False, f"seed item {L1_TITLE!r} missing" + # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). + logs = plane.work_items.work_logs.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 90 not in durations: + return False, f"no 90-minute work log on target item {wid}; durations={durations}" + + sum_rows: list[Any] = [] + try: + summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) + raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) + sum_rows = list(raw or []) + except Exception: + # Summary fetch is optional for person names; duration + title/summary/total still work. + sum_rows = [] + + final_text = _final_text(run) + if not _l1_duration_reported(final_text): + return False, "final text missing logged duration (numeric 90 or 1.5)" + if not _l1_summary_substance(final_text, title=L1_TITLE, sum_rows=sum_rows): + return False, ( + "final text lacks worklog summary substance " + "(need item title, person from summary, or words 'summary'/'total')" + ) + return True, f"90m log on {wid} + final text reports duration and summary substance" + + +async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L2: target has activities; final text reports the count via ``count: N`` contract.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L2_TITLE) + if not wid: + return False, f"seed item {L2_TITLE!r} missing" + try: + page = plane.work_items.activities.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except Exception as exc: + return False, f"activities.list failed: {exc}" + rows = page.results if hasattr(page, "results") else page + n = len(list(rows or [])) + if n < 1: + return False, "no activities on target (seed comments should create some)" + final_text = _final_text(run) + if not reports_contract_int(final_text, n): + return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + return True, f"final text reports activity count {n} via contract" + + +async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L5: final text reports the attachment count via ``count: N`` contract.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L5_TITLE) + if not wid: + return False, f"seed item {L5_TITLE!r} missing" + try: + page = plane.work_items.attachments.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except Exception as exc: + return False, f"attachments.list failed: {exc}" + rows = page.results if hasattr(page, "results") else page + n = len(list(rows or [])) + final_text = _final_text(run) + if not reports_contract_int(final_text, n): + return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + return True, f"final text reports attachment count {n} via contract" + + +async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L3: workspace has a release tag with version eval-rc1.""" + workspace_slug = ctx["workspace_slug"] + try: + page = plane.releases.tags.list(workspace_slug=workspace_slug) + except Exception as exc: + return False, f"list release tags failed: {exc}" + rows = page.results if hasattr(page, "results") else page + versions = {(getattr(t, "version", None) or "").strip() for t in (rows or [])} + if L3_TAG_VERSION in versions: + # Track for teardown if id available. + for t in rows or []: + if (getattr(t, "version", None) or "").strip() == L3_TAG_VERSION: + tid = getattr(t, "id", None) + if tid: + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "release_tag" and str(o.get("id")) == str(tid) for o in objs): + objs.append({"kind": "release_tag", "id": tid}) + break + return True, f"release tag {L3_TAG_VERSION!r} present" + return False, f"tag {L3_TAG_VERSION!r} missing; have {sorted(versions)}" + + +def _property_type_is_text(prop: Any) -> bool: + raw = getattr(prop, "property_type", None) + if raw is None: + raw = getattr(prop, "type", None) + if raw is None: + return False + if hasattr(raw, "value"): + raw = raw.value + if hasattr(raw, "name"): + # Enum member: PropertyType.TEXT + name = str(raw.name) + if name.upper() == "TEXT": + return True + s = str(raw).upper() + return s == "TEXT" or s.endswith(".TEXT") or s == "PROPERTYTYPE.TEXT" + + +async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L4: right customer has TEXT property 'Eval Industry' = Enterprise (exact name).""" + workspace_slug = ctx["workspace_slug"] + cust = ctx.get("customer") or {} + customer_id = cust.get("id") if isinstance(cust, dict) else cust + if not customer_id: + return False, "customer missing from seed" + try: + props = plane.customers.properties.list(workspace_slug=workspace_slug) + except Exception as exc: + return False, f"list customer properties failed: {exc}" + prop_rows = props.results if hasattr(props, "results") else props + target_prop: Any | None = None + for p in prop_rows or []: + # Exact display_name match only (case-insensitive full match) — not substring "Industry". + display = (getattr(p, "display_name", None) or "").strip() + if display.casefold() != L4_PROP_DISPLAY.casefold(): + continue + if not _property_type_is_text(p): + return False, ( + f"property {display!r} exists but property_type is not TEXT (got {getattr(p, 'property_type', None)!r})" + ) + target_prop = p + break + if target_prop is None: + return False, f"no TEXT customer property named exactly {L4_PROP_DISPLAY!r}" + pid = str(target_prop.id) + # Track for teardown. + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "customer_property" and str(o.get("id")) == pid for o in objs): + objs.append({"kind": "customer_property", "id": pid}) + try: + values = plane.customers.property_values.list(workspace_slug=workspace_slug, customer_id=customer_id) + except Exception as exc: + return False, f"get property values failed: {exc}" + if not isinstance(values, dict): + return False, f"unexpected property_values shape: {type(values)}" + vals = values.get(pid) or values.get(str(pid)) or [] + flat = [str(v) for v in (vals if isinstance(vals, list) else [vals])] + if any(L4_PROP_VALUE.casefold() == v.casefold() for v in flat): + return True, f"customer {customer_id} property {pid} ({L4_PROP_DISPLAY})={L4_PROP_VALUE!r}" + return False, f"customer {customer_id} property {pid} values {flat} lack {L4_PROP_VALUE!r}" + + +# --------------------------------------------------------------------------- +# Task catalog (full DESIGN list + extras for uncovered v2 families) +# --------------------------------------------------------------------------- + +TASKS: list[dict[str, Any]] = [ + { + "id": "R1", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, what is the current state of the work item titled " + f"'{R1_TITLE}'? Answer with the state name." + ), + "optimal_calls": 1, + "optimal_tools": {"list_work_items"}, + "alternate_tools": { + "search_work_items", + "list_archived_work_items", + "count_work_items", + "retrieve_work_item", + "retrieve_work_item_by_identifier", + "list_projects", + "list_states", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "list_states", + "get_workspace_context", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r1, + }, + { + "id": "R2", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, how many urgent open work items are there? Answer with the integer count only." + ), + "optimal_calls": 1, + "optimal_tools": {"count_work_items"}, + "alternate_tools": { + "list_work_items", + "search_work_items", + "list_projects", + "list_states", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + # No count tool on v2 — find_work_items with priority/state filters is optimal. + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "list_states", + "get_workspace_context", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r2, + }, + { + "id": "R3", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, list work items assigned to me that are due this week. Answer with their titles." + ), + "optimal_calls": 2, + "optimal_tools": {"get_me", "list_work_items"}, + "alternate_tools": { + "search_work_items", + "count_work_items", + "list_projects", + "get_workspace_members", + "get_pql_reference", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_workspace_context", + "get_work_item", + "search_projects", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r3, + }, + { + "id": "R4", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, what is in the active cycle, and is anything overdue? " + f"Name the cycle (expect '{CYCLE_CURRENT}') and any overdue item titles." + ), + "optimal_calls": 2, + "optimal_tools": {"list_cycles", "list_work_items"}, + "alternate_tools": { + "list_cycle_work_items", + "retrieve_cycle", + "search_work_items", + "list_projects", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "list_cycles", + "get_work_item", + "get_pql_reference", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"items", "cycles"}, + "verify": verify_r4, + }, + { + "id": "R5", + "tags": {"read", "tier1"}, + "prompt": ( + f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " + "Include the key phrases from its comments." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_comments"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "retrieve_work_item_by_identifier", + "list_work_item_activities", + "list_projects", + }, + "surface_tools": { + "v2": { + # include= depth: single get_work_item with include=comments after resolve, + # or find + get with include. + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "get_work_item"}, + "alternate_tools": { + "search_projects", + "get_workspace_context", + "create_comment", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r5, + }, + { + "id": "R6", + "tags": {"read", "tier1"}, + "prompt": ( + "Across the eval projects created for this run (main project {project} and its " + "sibling 'B' project), which project has more open Bug-typed work items? " + "Answer with the project name." + ), + "optimal_calls": 3, + "optimal_tools": {"list_projects", "list_work_items", "resolve_work_item_type"}, + "alternate_tools": { + "count_work_items", + "search_work_items", + "list_work_item_types", + "retrieve_project", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"search_projects", "find_work_items", "get_workspace_context"}, + "alternate_tools": { + "get_work_item", + "get_pql_reference", + "list_states", + }, + }, + # Type id resolution cleaner on v2-schema + "v2-schema": { + "optimal_calls": 3, + "optimal_tools": {"search_projects", "find_work_items", "resolve_work_item_type"}, + "alternate_tools": { + "list_work_item_types", + "get_workspace_context", + "get_work_item", + "get_pql_reference", + }, + }, + }, + "needs": {"items", "bug_type", "second_project"}, + "verify": verify_r6, + }, + { + "id": "W1", + "tags": {"write", "tier1"}, + "prompt": ( + "Create a work item in project {project}: title 'Login page 500s on empty " + "password', priority urgent, assign it to me, and add the 'auth' label." + ), + "optimal_calls": 4, + "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, + "alternate_tools": { + "search_work_items", + "list_states", + "retrieve_project", + "get_workspace_members", + "manage_work_item_assignee", + "manage_work_item_label", + "update_work_item", + "list_work_items", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"get_workspace_context", "create_work_item"}, + "alternate_tools": { + "search_projects", + "list_labels", + "find_work_items", + "get_work_item", + "update_work_item", + }, + }, + }, + "needs": {"labels"}, + "verify": verify_w1, + }, + { + "id": "W2", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, move the work item titled '{W2_TITLE}' to the Done state."), + "optimal_calls": 3, + "optimal_tools": {"list_work_items", "list_states", "update_work_item"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "retrieve_state", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "update_work_item"}, + "alternate_tools": { + "list_states", + "get_work_item", + "list_available_transitions", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w2, + }, + { + "id": "W3", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' " + "saying 'Reviewed contrast tokens — needs design pass'." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "create_work_item_comment"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "list_work_item_comments", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "create_comment"}, + "alternate_tools": { + "get_work_item", + "modify_comment", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w3, + }, + { + "id": "W4", + "tags": {"write", "tier1"}, + "prompt": ("In project {project}, rename the label 'triage' to 'needs-triage'."), + "optimal_calls": 2, + "optimal_tools": {"list_labels", "update_label"}, + "alternate_tools": { + "retrieve_label", + "create_label", + "delete_label", + "list_projects", + }, + "surface_tools": { + "v2": { + # Default v2 has list_labels but no update_label (schema tier). + "unsupported": True, + "reason": ("W4 needs update_label which is only on the v2-schema surface — use --surface v2-schema"), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": {"list_labels", "update_label"}, + "alternate_tools": { + "create_label", + "delete_label", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"labels"}, + "verify": verify_w4, + }, + { + "id": "W5", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, archive all completed work items in the module '{MODULE_NAME}'."), + "optimal_calls": 5, # list_modules + list_module_work_items + 3× archive + "optimal_tools": { + "list_modules", + "list_module_work_items", + "manage_work_item_archive", + }, + "alternate_tools": { + "list_work_items", + "retrieve_module", + "list_projects", + "list_states", + }, + "surface_tools": { + "v2": { + "optimal_calls": 5, + "optimal_tools": {"list_modules", "find_work_items", "archive_work_item"}, + "alternate_tools": { + "get_work_item", + "assign_to_module", + "search_projects", + "list_states", + }, + }, + }, + "needs": {"module"}, + "verify": verify_w5, + }, + { + "id": "W6", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, '{CYCLE_PAST}' is wrapping up. Close it and make sure " + f"its unfinished work items end up on '{CYCLE_CURRENT}'." + ), + "optimal_calls": 4, + "optimal_tools": { + "list_cycles", + "transfer_cycle_work_items", + "complete_cycle", + }, + "alternate_tools": { + "list_cycle_work_items", + "manage_cycle_work_items", + "update_cycle", + "list_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + # close_cycle with transfer_to is the consolidated path. + "optimal_calls": 2, + "optimal_tools": {"list_cycles", "close_cycle"}, + "alternate_tools": { + "assign_to_cycle", + "find_work_items", + "search_projects", + "get_workspace_context", + }, + }, + }, + # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — + # Plane rejects every edit to an ended cycle. See _seed_cycles. + "needs": {"items", "cycles", "cycles_open_past"}, + "verify": verify_w6, + }, + { + "id": "W7", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, mark the work item '{W7_SOURCE_TITLE}' as blocking " + f"'{W7_TARGET_TITLE}', and add the reference URL {W7_URL} on the blocking item." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_work_items", + "create_work_item_relation", + "create_work_item_link", + }, + "alternate_tools": { + "search_work_items", + "list_work_item_relations", + "list_work_item_relation_definitions", + "list_work_item_links", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"find_work_items", "link_work_items", "add_work_item_link"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "update_work_item", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w7, + }, + { + "id": "W8", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}' for yesterday."), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "create_work_log"}, + "alternate_tools": { + "search_work_items", + "list_work_logs", + "retrieve_work_item", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "log_work"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "update_work_item", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w8, + }, + { + "id": "W9", + "tags": {"write", "tier1", "extra"}, + "prompt": ( + "In project {project}, set priority to high on these three work items in one " + "batch: 'Checkout times out on 3DS challenge', " + "'Session cookie not rotated after login', " + "'Inventory count goes negative under load'." + ), + # Extra: exercises bulk_update_work_items (not in original DESIGN 20). + "optimal_calls": 4, + "optimal_tools": { + "list_work_items", + "update_work_item", + }, + "alternate_tools": { + "search_work_items", + "list_projects", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "bulk_update_work_items"}, + "alternate_tools": { + "update_work_item", + "get_work_item", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w9, + }, + { + "id": "W10", + "tags": {"write", "tier1", "extra"}, + "prompt": ( + "In project {project}, create a project page named 'Eval Runbook' with body " + "text 'Rollback steps for eval harness'." + ), + # Extra: exercises pages family (create_page / get_page). + "optimal_calls": 2, + "optimal_tools": {"list_projects", "create_page"}, + "alternate_tools": { + "list_pages", + "retrieve_page", + "attach_page_to_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"create_page"}, + "alternate_tools": { + "list_pages", + "get_page", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": set(), + "verify": verify_w10, + }, + { + "id": "S1", + "tags": {"setup", "tier1"}, + "prompt": ( + "In project {project}, add a Severity dropdown property (options: Critical, " + "Major, Minor) to the Bug work item type." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_projects", + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "create_work_item_property_option", + "retrieve_work_item_type", + "list_work_item_properties", + "retrieve_work_item_property", + "manage_work_item_type_properties", + "create_work_item_type", + "import_work_item_types_to_project", + "update_project_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S1 needs work-item-type/property schema tools " + "(resolve_work_item_type, create_work_item_property) which are " + "not on the default v2 surface — use --surface v2-schema" + ), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": { + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "list_work_item_properties", + "add_property_option", + "search_projects", + "get_workspace_context", + "get_features", + "configure_features", + "update_work_item_type", + }, + }, + }, + "needs": {"bug_type"}, + "verify": verify_s1, + }, + { + "id": "S2", + "tags": {"setup", "tier1"}, + "prompt": ( + f"In project {{project}}, add a Fibonacci estimate scale (points 1,2,3,5,8) " + f"and set the work item '{W8_TITLE}' to 5 points." + ), + "optimal_calls": 5, + "optimal_tools": { + "list_projects", + "create_project_estimate", + "create_project_estimate_points", + "link_estimate_to_project", + "update_work_item", + }, + "alternate_tools": { + "get_project_estimate", + "list_project_estimate_points", + "list_work_items", + "search_work_items", + "update_project_estimate", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S2 needs configure_estimate (schema tier) to create the Fibonacci " + "scale — default v2 has no estimate schema tools. " + "(v2 update_work_item does accept estimate_point.) Use v2-schema." + ), + }, + "v2-schema": { + # configure_estimate creates scale+points+link in one call; + # update_work_item(estimate_point="5") resolves the value server-side. + "optimal_calls": 2, + "optimal_tools": {"configure_estimate", "update_work_item"}, + "alternate_tools": { + "search_projects", + "get_features", + "find_work_items", + "get_work_item", + "get_workspace_context", + "bulk_update_work_items", + }, + }, + }, + "needs": {"items"}, + "verify": verify_s2, + }, + { + "id": "S3", + "tags": {"setup", "tier1"}, + "prompt": ( + "In project {project}, create a work item type named 'Incident' and add a " + "required text property (e.g. 'Impact summary') on it." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_projects", + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "create_work_item_type", + "list_work_item_types", + "import_work_item_types_to_project", + "list_work_item_properties", + "manage_work_item_type_properties", + "update_project_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S3 needs resolve_work_item_type + create_work_item_property " + "(schema tier) — use --surface v2-schema" + ), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": { + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "list_work_item_properties", + "update_work_item_type", + "search_projects", + "get_features", + "configure_features", + }, + }, + }, + "needs": set(), + "verify": verify_s3, + }, + { + "id": "S4", + "tags": {"setup", "tier1"}, + "prompt": ( + f"In project {{project}}, triage intake: accept the billing request " + f"'{INTAKE_BILLING_TITLE}' and reject/decline the spam item " + f"'{INTAKE_SPAM_TITLE}'." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_intake_work_items", + "update_intake_work_item", + }, + "alternate_tools": { + "retrieve_intake_work_item", + "list_work_items", + "list_projects", + "create_intake_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"list_intake", "triage_intake"}, + "alternate_tools": { + "find_work_items", + "get_work_item", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"intake"}, + "verify": verify_s4, + }, + { + "id": "S5", + "tags": {"setup", "tier1"}, + "prompt": ( + "Enable cycles and time tracking (worklogs) for project {project}, " + "and enable the customers feature for the workspace." + ), + # Minimal legacy path (2 calls): + # 1. update_project(cycle_view=True, is_time_tracking_enabled=True) + # 2. update_workspace_features(customers=True) + # (features PATCH can set cycles→cycle_view but cannot set worklogs.) + "optimal_calls": 2, + "optimal_tools": {"update_project", "update_workspace_features"}, + "alternate_tools": { + "update_project_features", + "list_projects", + "retrieve_project", + "get_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S5 needs configure_features (schema tier) for project cycles/worklogs " + "and workspace customers — use --surface v2-schema" + ), + }, + "v2-schema": { + # 2 calls: configure_features(project, cycles+worklogs) + + # configure_features(customers=True) without project. + "optimal_calls": 2, + "optimal_tools": {"configure_features"}, + "alternate_tools": { + "get_features", + "search_projects", + "get_workspace_context", + "update_work_item", + }, + }, + }, + # Seed leaves project cycles+worklogs and workspace customers off. + "needs": {"leave_cycles_worklogs_off"}, + "verify": verify_s5, + }, + { + "id": "C1", + "tags": {"write", "tier1"}, + "prompt": ( + f"Create customer '{CUSTOMER_NAME}' (if it does not already exist), add a " + f"request named '{CUSTOMER_REQUEST_NAME}', and link that request to the work " + f"item '{R1_TITLE}' in project {{project}}." + ), + "optimal_calls": 4, + "optimal_tools": { + "list_customers", + "create_customer", + "create_customer_request", + "list_work_items", + }, + "alternate_tools": { + "retrieve_customer", + "manage_customer_work_items", + "list_customer_requests", + "list_customer_work_items", + "search_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 4, + "optimal_tools": { + "list_customers", + "create_customer", + "log_customer_request", + "link_customer_work_items", + }, + "alternate_tools": { + "get_customer", + "find_work_items", + "update_customer", + "search_projects", + }, + }, + }, + # No pre-seeded customer — agent creates; items needed for link target. + "needs": {"items"}, + "verify": verify_c1, + }, + { + "id": "C2", + "tags": {"read", "tier1"}, + "prompt": (f"What shipped in release {RELEASE_NAME}? Summarize the changelog."), + "optimal_calls": 2, + "optimal_tools": {"list_releases", "get_release_changelog"}, + "alternate_tools": { + "retrieve_release", + "list_release_work_items", + "update_release_changelog", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"get_release"}, + "alternate_tools": { + "list_releases", + "assign_to_release", + "get_workspace_context", + }, + }, + }, + "needs": {"release"}, + "verify": verify_c2, + }, + { + "id": "R7", + "tags": {"read", "tier1", "extra"}, + "prompt": ( + f"In project {{project}}, what states can the work item '{R1_TITLE}' " + "legally transition to under workflow rules? List the state names " + "(or say unrestricted if none)." + ), + # Extra: exercises list_available_transitions. + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_states"}, + "alternate_tools": { + "retrieve_work_item", + "search_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"list_available_transitions"}, + "alternate_tools": { + "find_work_items", + "get_work_item", + "list_states", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r7, + }, + # ------------------------------------------------------------------ + # ID-in-hand class (I*): identifiers handed in the prompt — no name resolution advantage + # ------------------------------------------------------------------ + { + "id": "I1", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, update work item {work_item_id}: set its priority to high."), + "prompt_bind": _bind_item_uuid(I1_TITLE), + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + "retrieve_work_item_by_identifier", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "find_work_items", "search_projects"}, + }, + }, + "needs": {"items"}, + "verify": verify_i1, + }, + { + "id": "I2", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "id_in_hand", "debias"}, + "prompt": ( + "In project {project}, what is the current state of work item " + "{work_item_identifier}? Answer with the state name only." + ), + "prompt_bind": _bind_item_identifier(I2_TITLE), + "optimal_calls": 1, + "optimal_tools": {"retrieve_work_item_by_identifier"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + "list_states", + }, + "surface_tools": { + "v2": { + # get_work_item requires UUIDs (forwards work_item_id directly). + # PROJ-N on v2 is resolved via find_work_items (list/filter). + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "list_states", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"items"}, + "verify": verify_i2, + }, + { + "id": "I3", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, add work item {work_item_id} to cycle {cycle_id}."), + "prompt_bind": _bind_i3, + "optimal_calls": 1, + "optimal_tools": {"manage_cycle_work_items"}, + "alternate_tools": { + "list_cycles", + "list_cycle_work_items", + "list_work_items", + "retrieve_cycle", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"assign_to_cycle"}, + "alternate_tools": {"list_cycles", "find_work_items", "get_work_item"}, + }, + }, + "needs": {"items", "cycles"}, + "verify": verify_i3, + }, + { + "id": "I4", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, attach label {label_id} to work item {work_item_id}."), + "prompt_bind": _bind_i4, + "optimal_calls": 1, + "optimal_tools": {"manage_work_item_label"}, + "alternate_tools": { + "update_work_item", + "list_labels", + "retrieve_work_item", + "list_work_items", + }, + "surface_tools": { + "v2": { + # Default v2 update_work_item accepts labels; no manage_work_item_label. + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "list_labels", "find_work_items"}, + }, + }, + "needs": {"items", "labels"}, + "verify": verify_i4, + }, + { + "id": "I5", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, set the priority of work item {work_item_id} to low."), + "prompt_bind": _bind_item_uuid(I3_TITLE), # footer item; not high-traffic elsewhere + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "find_work_items"}, + }, + }, + "needs": {"items"}, + "verify": verify_i5, + }, + # ------------------------------------------------------------------ + # Long-tail class (L*): tools outside the default v2 curated surface + # ------------------------------------------------------------------ + { + "id": "L1", + "author": "post-hoc-debias", + "tags": {"write", "read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " + f"'{L1_TITLE}', then report the project's worklog summary (who/what has time logged)." + ), + "optimal_calls": 3, + "optimal_tools": {"list_work_items", "create_work_log", "get_project_worklog_summary"}, + "alternate_tools": { + "search_work_items", + "list_work_logs", + "retrieve_work_item", + "list_projects", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ( + "L1 needs get_project_worklog_summary (legacy project tool) — not on the default v2 surface" + ), + }, + }, + "needs": {"items"}, + "verify": verify_l1, + }, + { + "id": "L2", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, list the activity history for the work item titled " + f"'{L2_TITLE}'. Summarize how many activities there are and mention any " + "notable comment phrases you see. End your answer with a line of the form " + "'count: N' where N is the number of activities." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_activities"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "list_work_item_comments", + "retrieve_work_item_activity", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ( + "L2 needs list_work_item_activities — not on the default v2 surface " + "(v2 has comments include= but not the activities feed)" + ), + }, + }, + "needs": {"items", "activity_feed"}, + "verify": verify_l2, + }, + { + "id": "L3", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "long_tail", "debias"}, + "prompt": (f"Create a release tag with version '{L3_TAG_VERSION}' (a version marker for the eval run)."), + "optimal_calls": 1, + "optimal_tools": {"create_release_tag"}, + "alternate_tools": { + "list_release_tags", + "retrieve_release_tag", + "list_releases", + "update_release_tag", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": "L3 needs create_release_tag — not on the default v2 surface", + }, + }, + "needs": set(), # workspace-level tag; no project fixture required + "verify": verify_l3, + }, + { + "id": "L4", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "long_tail", "debias"}, + "prompt": ( + f"For customer '{CUSTOMER_NAME}', ensure there is a text customer property " + f"named '{L4_PROP_DISPLAY}' and set its value to '{L4_PROP_VALUE}'." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_customers", + "create_customer_property", + "set_customer_property_values", + }, + "alternate_tools": { + "list_customer_properties", + "get_customer_property_values", + "retrieve_customer", + "update_customer_property", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ( + "L4 needs create_customer_property / set_customer_property_values — not on the default v2 surface" + ), + }, + }, + "needs": {"customer"}, + "verify": verify_l4, + }, + { + "id": "L5", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, how many file attachments does the work item titled " + f"'{L5_TITLE}' have? End your answer with a line of the form 'count: N' " + "where N is the number of file attachments." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_attachments"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "get_work_item_attachment_download_url", + }, + "surface_tools": { + "v2": { + # Achievable on default v2 via include=attachments on get_work_item. + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "get_work_item"}, + "alternate_tools": { + "search_projects", + "get_workspace_context", + "list_states", + }, + }, + }, + "needs": {"items"}, + "verify": verify_l5, + }, +] + + +def resolve_surface_tool_sets( + task: dict[str, Any], + surface: str, +) -> dict[str, Any]: + """Resolve optimal/alternate tool sets for a surface. + + Returns a dict with: + - skip (str | None): if set, the runner should SKIP the task on this surface + - optimal_tools / alternate_tools: classification sets + - optimal_calls: optional override + - classification: ``exact`` when an overlay or full/legacy sets apply; + ``approximate`` when falling back to flat legacy-named sets on a non-full + surface that has no overlay + """ + surface = (surface or "full").strip().lower() + overlays = task.get("surface_tools") or {} + + if surface in ("full", "legacy", ""): + return { + "skip": None, + "optimal_tools": set(task["optimal_tools"]), + "alternate_tools": set(task["alternate_tools"]), + "optimal_calls": task.get("optimal_calls"), + "classification": "exact", + } + + ov = overlays.get(surface) + # v2-schema is a superset of v2 for *supported* tools, but schema adds none of + # the long-tail APIs (worklog summary, activities, release tags, customer + # property values). Inherit the full v2 overlay — including expected_skip / + # unsupported — when no schema-specific entry exists. + if ov is None and surface == "v2-schema": + ov = overlays.get("v2") + + if ov is None: + return { + "skip": None, + "optimal_tools": set(task["optimal_tools"]), + "alternate_tools": set(task["alternate_tools"]), + "optimal_calls": task.get("optimal_calls"), + "classification": "approximate", + } + + if ov.get("unsupported") or ov.get("expected_skip"): + return { + "skip": ov.get("reason") or f"task {task.get('id')} unsupported on surface {surface}", + "optimal_tools": set(), + "alternate_tools": set(), + "optimal_calls": None, + "classification": "exact", + } + + optimal = set(ov["optimal_tools"]) + alternate = set(ov["alternate_tools"]) + if not optimal.isdisjoint(alternate): + raise ValueError(f"{task.get('id')}/{surface}: optimal/alternate overlap") + return { + "skip": None, + "optimal_tools": optimal, + "alternate_tools": alternate, + "optimal_calls": ov.get("optimal_calls", task.get("optimal_calls")), + "classification": "exact", + } + + +TASKS_BY_ID: dict[str, dict[str, Any]] = {t["id"]: t for t in TASKS} + + +def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: + """Return tasks filtered by id list (None = all).""" + if ids is None: + return list(TASKS) + missing = [i for i in ids if i not in TASKS_BY_ID] + if missing: + raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") + return [TASKS_BY_ID[i] for i in ids] + + +def task_author(task: dict[str, Any]) -> str: + """Return the task author; default ``claude`` when the key is absent.""" + return str(task.get("author") or "claude") + + +def _serialize_surface_tools(surface_tools: dict[str, Any] | None) -> dict[str, Any]: + """Stable JSON-friendly form of a task's surface_tools overlay.""" + if not surface_tools: + return {} + out: dict[str, Any] = {} + for surface in sorted(surface_tools): + ov = surface_tools[surface] or {} + if not isinstance(ov, dict): + out[surface] = ov + continue + entry: dict[str, Any] = {} + for key in sorted(ov): + val = ov[key] + if isinstance(val, set | frozenset): + entry[key] = sorted(val) + elif isinstance(val, list | tuple): + entry[key] = list(val) + else: + entry[key] = val + out[surface] = entry + return out + + +def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: + """Stable short hash of the task battery used for a run. + + SHA-256 (first 12 hex chars) over a canonical serialization of every task + sorted by id: id, prompt, sorted optimal/alternate tools, optimal_calls, + and the surface_tools overlay (sets sorted, keys sorted). + + Ceilings (intentionally *not* covered by the hash): + - Verifier functions and ``needs`` fixtures do not alter the fingerprint — + prompt/tool-set drift is the stability signal, not seed/verify logic. + - The hash covers the *selected* task list: ``--tasks`` subsets produce + different fingerprints than a full-catalog run. + """ + src = list(TASKS if tasks is None else tasks) + payload: list[dict[str, Any]] = [] + for t in sorted(src, key=lambda x: str(x.get("id") or "")): + payload.append( + { + "id": t.get("id"), + "prompt": t.get("prompt"), + "optimal_tools": sorted(t.get("optimal_tools") or []), + "alternate_tools": sorted(t.get("alternate_tools") or []), + "optimal_calls": t.get("optimal_calls"), + "surface_tools": _serialize_surface_tools(t.get("surface_tools")), + } + ) + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] diff --git a/pyproject.toml b/pyproject.toml index cfe84dac..8b301b02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,10 @@ dev = [ "pytest>=7.0.0", "ruff>=0.1.0", ] +# Only the `sdk` eval driver needs this; the CLI drivers shell out to an agent. +evals = [ + "anthropic[mcp]>=0.121.0", +] [project.scripts] plane-mcp-server = "plane_mcp.__main__:main" diff --git a/tests/test_evals_catalog.py b/tests/test_evals_catalog.py new file mode 100644 index 00000000..45cb1b12 --- /dev/null +++ b/tests/test_evals_catalog.py @@ -0,0 +1,609 @@ +"""Offline tests for the full eval task catalog + seed plan + verifiers.""" + +from __future__ import annotations + +import inspect + +import pytest + +from evals import seed as seed_mod +from evals import tasks as tasks_mod +from evals.run import cmd_dry_run, cmd_list, parse_args +from evals.seed import seed_plan +from evals.tasks import TASKS, TASKS_BY_ID, get_tasks, resolve_surface_tool_sets + +# DESIGN.md catalog ids (stable) + extras added for uncovered v2 families. +DESIGN_IDS = { + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "S1", + "S2", + "S3", + "S4", + "C1", + "C2", +} +EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features +# WS3 de-biasing classes +ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} +LONG_TAIL_IDS = {"L1", "L2", "L3", "L4", "L5"} +# Workspace-scoped prompts that omit {project} +NO_PROJECT_PROMPT_IDS = {"C2", "L3", "L4"} + + +@pytest.fixture(autouse=True) +def _eval_creds(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +def test_catalog_includes_design_and_extras(): + ids = {t["id"] for t in TASKS} + assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" + assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" + assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" + assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" + assert len(TASKS) >= 20 + + +def test_get_tasks_all_and_filter(): + all_t = get_tasks(None) + assert len(all_t) == len(TASKS) + subset = get_tasks(["R1", "W9", "C2"]) + assert [t["id"] for t in subset] == ["R1", "W9", "C2"] + + +def test_get_tasks_unknown_exits(): + with pytest.raises(SystemExit): + get_tasks(["NOPE"]) + + +def test_task_schema_invariants(): + for t in TASKS: + assert t["id"] + assert isinstance(t["tags"], set) + assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS + assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] + assert isinstance(t["alternate_tools"], set) + assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] + assert callable(t["verify"]) + assert isinstance(t.get("needs"), set) + # Overlays: optimal/alternate disjoint when present (skip capability-gap marks) + for surface, ov in (t.get("surface_tools") or {}).items(): + if ov.get("unsupported") or ov.get("expected_skip"): + continue + opt = set(ov["optimal_tools"]) + alt = set(ov["alternate_tools"]) + assert opt.isdisjoint(alt), f"{t['id']}/{surface} overlap" + + +def test_debias_tasks_author_and_v2_skips(): + from evals.tasks import task_author + + for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: + t = TASKS_BY_ID[tid] + assert task_author(t) == "post-hoc-debias" + # L1–L4 are capability gaps on default v2 (+ inherited by v2-schema). + for tid in ("L1", "L2", "L3", "L4"): + for surface in ("v2", "v2-schema"): + skip = resolve_surface_tool_sets(TASKS_BY_ID[tid], surface)["skip"] + assert skip, f"{tid} should expected_skip on {surface}" + assert resolve_surface_tool_sets(TASKS_BY_ID[tid], "full")["skip"] is None + # L5 is achievable on v2 via get_work_item(include=attachments). + assert resolve_surface_tool_sets(TASKS_BY_ID["L5"], "v2")["skip"] is None + l5 = resolve_surface_tool_sets(TASKS_BY_ID["L5"], "v2") + assert l5["optimal_tools"] == {"find_work_items", "get_work_item"} + assert l5["optimal_calls"] == 2 + # I-class is runnable on v2 (raw call efficiency, not a capability gap) + for tid in ID_IN_HAND_IDS: + assert resolve_surface_tool_sets(TASKS_BY_ID[tid], "v2")["skip"] is None + # I2: PROJ-N is find_work_items, not get_work_item (UUID-only). + i2 = resolve_surface_tool_sets(TASKS_BY_ID["I2"], "v2") + assert i2["optimal_tools"] == {"find_work_items"} + assert "get_work_item" not in i2["optimal_tools"] + + +def test_v2_schema_inherits_expected_skip(): + """v2-schema must inherit L1 expected_skip (schema adds none of those APIs).""" + l1_v2 = resolve_surface_tool_sets(TASKS_BY_ID["L1"], "v2") + l1_schema = resolve_surface_tool_sets(TASKS_BY_ID["L1"], "v2-schema") + assert l1_v2["skip"] + assert l1_schema["skip"] == l1_v2["skip"] + # S1 remains schema-supported (explicit v2-schema overlay). + assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2")["skip"] + assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema")["skip"] is None + + +def test_every_task_resolves_on_full_v2_v2schema(): + for t in TASKS: + for surface in ("full", "v2", "v2-schema"): + out = resolve_surface_tool_sets(t, surface) + assert "classification" in out + assert out["classification"] in {"exact", "approximate"} + # skip is either None or a non-empty reason string + if out["skip"] is not None: + assert isinstance(out["skip"], str) and out["skip"] + + +def test_s1_w4_s2_s3_surface_skips(): + assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2")["skip"] + assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema")["skip"] is None + assert resolve_surface_tool_sets(TASKS_BY_ID["W4"], "v2")["skip"] + assert resolve_surface_tool_sets(TASKS_BY_ID["W4"], "v2-schema")["skip"] is None + assert resolve_surface_tool_sets(TASKS_BY_ID["S2"], "v2")["skip"] + # S2 became supported on v2-schema once update_work_item gained estimate_point. + s2 = resolve_surface_tool_sets(TASKS_BY_ID["S2"], "v2-schema") + assert s2["skip"] is None + assert s2["optimal_tools"] == {"configure_estimate", "update_work_item"} + assert s2["optimal_calls"] == 2 + assert resolve_surface_tool_sets(TASKS_BY_ID["S3"], "v2")["skip"] + assert resolve_surface_tool_sets(TASKS_BY_ID["S3"], "v2-schema")["skip"] is None + assert resolve_surface_tool_sets(TASKS_BY_ID["S5"], "v2")["skip"] + s5 = resolve_surface_tool_sets(TASKS_BY_ID["S5"], "v2-schema") + assert s5["skip"] is None + assert s5["optimal_tools"] == {"configure_features"} + assert s5["optimal_calls"] == 2 + full_s5 = resolve_surface_tool_sets(TASKS_BY_ID["S5"], "full") + assert full_s5["optimal_tools"] == {"update_project", "update_workspace_features"} + assert full_s5["optimal_calls"] == 2 + + +def test_v2_schema_inherits_v2_overlay_when_absent(): + """v2-schema with no own overlay uses the v2 sets (superset surface).""" + r1 = resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2-schema") + assert r1["skip"] is None + assert r1["classification"] == "exact" + assert r1["optimal_tools"] == {"find_work_items"} + # S1 has an explicit v2-schema overlay (not the unsupported v2 one). + s1 = resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema") + assert s1["skip"] is None + assert "create_work_item_property" in s1["optimal_tools"] + + +def test_w6_seeds_an_open_cycle(): + """W6 asks the agent to close Sprint 12, so the seed must leave it open. + + Plane rejects every edit to an ended cycle, so a pre-closed fixture makes the + task unachievable by design. + """ + assert "cycles_open_past" in TASKS_BY_ID["W6"]["needs"] + assert "cycles" in TASKS_BY_ID["W6"]["needs"] + + +def test_v2_overlays_use_v2_tool_names(): + """Spot-check headline v2 paths.""" + assert resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2")["optimal_tools"] == {"find_work_items"} + w6 = resolve_surface_tool_sets(TASKS_BY_ID["W6"], "v2") + assert "close_cycle" in w6["optimal_tools"] + s4 = resolve_surface_tool_sets(TASKS_BY_ID["S4"], "v2") + assert "triage_intake" in s4["optimal_tools"] and "list_intake" in s4["optimal_tools"] + w9 = resolve_surface_tool_sets(TASKS_BY_ID["W9"], "v2") + assert "bulk_update_work_items" in w9["optimal_tools"] + w10 = resolve_surface_tool_sets(TASKS_BY_ID["W10"], "v2") + assert "create_page" in w10["optimal_tools"] + r7 = resolve_surface_tool_sets(TASKS_BY_ID["R7"], "v2") + assert "list_available_transitions" in r7["optimal_tools"] + c2 = resolve_surface_tool_sets(TASKS_BY_ID["C2"], "v2") + assert "get_release" in c2["optimal_tools"] + c1 = resolve_surface_tool_sets(TASKS_BY_ID["C1"], "v2") + assert "create_customer" in c1["optimal_tools"] + + +def test_seed_plan_covers_all_groups(): + groups = { + "items", + "labels", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + } + lines = seed_plan(groups) + blob = "\n".join(lines) + for g in groups: + assert ( + g.split("_")[0] in blob or g in blob or g.replace("_", " ") in blob or any(g in line for line in lines) + ), f"seed_plan missing {g}: {lines}" + # Specific fixtures named + assert "Sprint 12" in blob + assert "Checkout revamp" in blob + assert "1.2.0" in blob + assert "Acme Corp" in blob + + +def test_seed_plan_empty_needs_only_project(): + lines = seed_plan(set()) + assert any("project" in line for line in lines) + # project line + default workspace customers enable note + assert any("customers" in line for line in lines) + assert len(lines) == 2 + + +def test_verifiers_are_async_and_importable(): + for t in TASKS: + fn = t["verify"] + assert inspect.iscoroutinefunction(fn), t["id"] + # Callables resolve without NameError + assert fn.__module__ == "evals.tasks" + + +def test_cmd_list_prints_all_task_ids(capsys): + rc = cmd_list() + assert rc == 0 + out = capsys.readouterr().out + for tid in DESIGN_IDS | EXTRA_IDS: + assert tid in out + + +def test_cmd_dry_run_all_tasks(capsys): + rc = cmd_dry_run(list(TASKS)) + assert rc == 0 + out = capsys.readouterr().out + assert "Seed plan:" in out + for tid in ("R1", "W9", "S4", "C2", "R7"): + assert f"=== {tid} ===" in out + + +def test_parse_args_list(): + a = parse_args(["--list"]) + assert a.list is True + + +def test_tasks_module_has_no_hardcoded_uuids(): + """Regression: verifiers must resolve expected values at verify time.""" + src = inspect.getsource(tasks_mod) + # Crude: no UUID-shaped literals in tasks module. + assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) + + +def test_seed_module_ast_has_all_group_handlers(): + """seed() dispatches every documented fixture group.""" + src = inspect.getsource(seed_mod.seed) + for group in ( + "labels", + "items", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + ): + assert f'"{group}"' in src or f"'{group}'" in src, group + + +def test_seed_enables_project_features_immediately_after_create(monkeypatch): + """Fresh projects ship with cycles/modules/intake/worklogs off — seed must enable them. + + Sequence: create → workspace features (customers) → project update → project features. + """ + from types import SimpleNamespace + + from plane.models.projects import ProjectFeature, UpdateProject + from plane.models.workspaces import WorkspaceFeature + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + calls.append(("create", workspace_slug, getattr(data, "name", None))) + return SimpleNamespace(id="proj-main") + + def update(self, workspace_slug, project_id, data): + assert isinstance(data, UpdateProject) + calls.append(("update", project_id, data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + assert isinstance(data, ProjectFeature) + calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(("ws_update_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx) + + assert ctx["project_id"] == "proj-main" + kinds = [c[0] for c in calls] + assert kinds == ["create", "ws_update_features", "update", "update_features"] + # Workspace customers enabled for C1 preconditions + assert calls[1][1].get("customers") is True + assert "work_item_types" not in calls[1][1] + # Project enable calls target the created id + assert calls[2][1] == "proj-main" + assert calls[3][1] == "proj-main" + upd = calls[2][2] + assert upd.get("cycle_view") is True + assert upd.get("is_time_tracking_enabled") is True + feat = calls[3][2] + assert feat.get("cycles") is True + + +def test_seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): + """S5 needs leave_cycles_worklogs_off — project cycles/worklogs + workspace customers OFF.""" + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-s5") + + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + calls.append(("ws_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) + assert ctx["feature_exclude"] == ["cycles", "worklogs"] + assert ctx["ws_feature_exclude"] == ["customers"] + assert ctx["s5_left_customers_off"] is True + # No workspace customers enable call (excluded → _enable_workspace_features no-ops) + assert not any(c[0] == "ws_features" for c in calls) + upd = next(c[1] for c in calls if c[0] == "update") + assert "cycle_view" not in upd + assert "is_time_tracking_enabled" not in upd + assert upd.get("module_view") is True + feat = next(c[1] for c in calls if c[0] == "features") + assert "cycles" not in feat + assert feat.get("modules") is True + + +def test_seed_cycles_create_add_then_backdate(monkeypatch): + """Sprint 12: create (active end) → add_work_items → update(end_date past). + + Plane rejects adds when end_date is already past; seed must not create past first. + """ + from types import SimpleNamespace + + from plane.models.cycles import CreateCycle, UpdateCycle + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + cycle_seq = {"n": 0} + + class _Cycles: + def create(self, workspace_slug, project_id, data): + assert isinstance(data, CreateCycle) + cycle_seq["n"] += 1 + cid = f"cyc-{cycle_seq['n']}" + calls.append( + ( + "create", + { + "name": data.name, + "start_date": data.start_date, + "end_date": data.end_date, + "id": cid, + }, + ) + ) + return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) + + def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): + calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) + + def update(self, workspace_slug, project_id, cycle_id, data): + assert isinstance(data, UpdateCycle) + calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) + return SimpleNamespace(id=cycle_id, end_date=data.end_date) + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-1") + + def update(self, workspace_slug, project_id, data): + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + return data + + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {}) + + class _Users: + def get_me(self): + return SimpleNamespace(id="user-1") + + class _States: + def list(self, workspace_slug, project_id): + return SimpleNamespace( + results=[ + SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), + ] + ) + + item_n = {"n": 0} + + class _WorkItems: + def create(self, workspace_slug, project_id, data): + item_n["n"] += 1 + return SimpleNamespace( + id=f"wi-{item_n['n']}", + name=data.name, + state="st-started", + created_at="2026-01-01", + ) + + def update(self, workspace_slug, project_id, work_item_id, data): + return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) + + class comments: + @staticmethod + def create(**kw): + return SimpleNamespace(id="c1") + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + cycles=_Cycles(), + users=_Users(), + states=_States(), + work_items=_WorkItems(), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) + + # Filter to Sprint-12-related create/add/update sequence (first cycle is past). + past_id = ctx["cycle_past_id"] + # Must create both cycles before any backdate update of past. + create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] + assert len(create_idxs) == 2 + past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) + # Created with temporary *future* end_date (active), not the final past end. + assert past_create[1]["end_date"] > past_create[1]["start_date"] + # At least one add to past cycle before its update + past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] + past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] + assert past_adds, "expected add_work_items on Sprint 12" + assert past_updates, "expected backdate update on Sprint 12" + assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" + # Backdated end matches W6 seed ctx; differs from create-time active end + backdated_end = calls[past_updates[0]][1]["end_date"] + assert ctx["cycle_past_seed_end_date"] == backdated_end + assert backdated_end != past_create[1]["end_date"] + assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] + # Active cycle: create with future end; never backdated + cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) + assert cur_create[1]["end_date"] + cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] + assert cur_updates == [] + + +def test_teardown_s5_reenables_workspace_customers(monkeypatch): + from types import SimpleNamespace + + from plane.models.workspaces import WorkspaceFeature + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list = [] + + class _Workspaces: + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(data.model_dump(exclude_none=True)) + return data + + plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) + seed_mod.teardown(plane, {"workspace_slug": "test-ws", "s5_left_customers_off": True, "project_id": None}) + assert calls and calls[0].get("customers") is True + + +def test_seed_enables_features_on_second_project_too(monkeypatch): + """R6 second project also gets feature enable after its create.""" + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + creates: list[str] = [] + enables: list[str] = [] + + class _Projects: + def create(self, workspace_slug, data): + pid = f"p-{len(creates)}" + creates.append(pid) + return SimpleNamespace(id=pid) + + def update(self, workspace_slug, project_id, data): + enables.append(("update", project_id)) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + enables.append(("features", project_id)) + return data + + # Minimal stubs so second_project seed gets past bug_type + work items. + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) + + def update_features(self, workspace_slug, data): + enables.append(("ws_features", workspace_slug)) + return data + + class _WorkItemTypes: + def list(self, **kw): + return [SimpleNamespace(id="bug-1", name="Bug")] + + def create(self, **kw): + return SimpleNamespace(id="bug-1", name="Bug") + + def import_to_project(self, **kw): + return None + + class _WorkItems: + def create(self, **kw): + return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + work_item_types=_WorkItemTypes(), + work_items=_WorkItems(), + ) + ctx: dict = {} + # second_project path also seeds bug_type when missing + seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) + + assert len(creates) == 2 + # Each create followed by update + update_features for that project id + assert ("update", creates[0]) in enables + assert ("features", creates[0]) in enables + assert ("update", creates[1]) in enables + assert ("features", creates[1]) in enables diff --git a/tests/test_evals_debias_verifiers.py b/tests/test_evals_debias_verifiers.py new file mode 100644 index 00000000..623a1fe9 --- /dev/null +++ b/tests/test_evals_debias_verifiers.py @@ -0,0 +1,1031 @@ +"""Adversarial offline verifier tests for WS3 de-bias tasks + sample existing. + +For each covered verifier: (a) untouched seed end-state must FAIL, and +(b) a plausibly-wrong end state (right field wrong value / right value wrong +item) must FAIL. Fake plane clients only — no network. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.seed import R1_TITLE, W2_TITLE, W8_TITLE +from evals.tasks import ( + I1_TITLE, + I3_TITLE, + I4_TITLE, + L1_TITLE, + L2_TITLE, + L3_TAG_VERSION, + L4_PROP_DISPLAY, + L4_PROP_VALUE, + L5_TITLE, + verify_c2, + verify_i1, + verify_i2, + verify_i3, + verify_i4, + verify_i5, + verify_l1, + verify_l2, + verify_l3, + verify_l4, + verify_l5, + verify_r1, + verify_w2, + verify_w4, + verify_w8, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str = "") -> dict[str, Any]: + return {"final_text": text, "calls": []} + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +@pytest.fixture(autouse=True) +def _no_redis(monkeypatch): + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +# --------------------------------------------------------------------------- +# Shared fakes +# --------------------------------------------------------------------------- + + +class _WIRetrievePlane: + """work_items.retrieve + list by name; optional labels expand.""" + + def __init__( + self, + *, + by_id: dict[str, Any], + by_name: dict[str, str] | None = None, + states: list[Any] | None = None, + ): + self._by_id = by_id + self._by_name = by_name or {} + self._states = states or [] + self.work_items = SimpleNamespace( + list=self._list, + retrieve=self._retrieve, + ) + self.states = SimpleNamespace(list=lambda **kw: _Page(self._states)) + + def _list(self, **kw): + # Minimal name filter support used by _find_item_by_name. + params = kw.get("params") + name = None + if params is not None: + name = getattr(params, "name", None) or (params.get("name") if isinstance(params, dict) else None) + if name and name in self._by_name: + wid = self._by_name[name] + row = self._by_id.get(wid) or _item(wid, name) + return _Page([row]) + return _Page([]) + + def _retrieve(self, **kw): + wid = str(kw["work_item_id"]) + if wid not in self._by_id: + raise LookupError(wid) + return self._by_id[wid] + + +# --------------------------------------------------------------------------- +# I1 — priority high by UUID +# --------------------------------------------------------------------------- + + +def test_i1_untouched_urgent_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} + ok, note = await verify_i1(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i1_wrong_item_high_target_still_urgent_fails(): + async def _go(): + # Right value on the wrong item; target remains urgent. + plane = _WIRetrievePlane( + by_id={ + "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), + "wi-other": SimpleNamespace(id="wi-other", priority="high"), + } + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} + ok, note = await verify_i1(plane, ctx, _run()) + assert ok is False, note + assert "urgent" in note or "high" in note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# I2 — state by identifier in final text +# --------------------------------------------------------------------------- + + +def test_i2_untouched_empty_final_text_fails(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[st], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i2_wrong_state_name_in_text_fails(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[ + st, + SimpleNamespace(id="st-done", name="Done", group="completed"), + ], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("Done")) + assert ok is False, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# I3 — cycle membership by UUIDs +# --------------------------------------------------------------------------- + + +class _I3Plane: + def __init__(self, cycle_item_ids: list[str]): + self.cycles = SimpleNamespace(list_work_items=lambda **kw: _Page([_item(i, f"n-{i}") for i in cycle_item_ids])) + + +def test_i3_untouched_not_on_cycle_fails(): + async def _go(): + plane = _I3Plane(["other-1", "other-2"]) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i3_wrong_item_on_cycle_target_missing_fails(): + async def _go(): + plane = _I3Plane(["wrong-item"]) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# I4 — label attach by UUIDs +# --------------------------------------------------------------------------- + + +def test_i4_untouched_no_label_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[])}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i4_wrong_label_attached_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[SimpleNamespace(id="lab-auth")])}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# I5 — priority low by UUID +# --------------------------------------------------------------------------- + + +def test_i5_untouched_none_priority_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="none")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i5_wrong_value_high_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="high")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, note + assert "high" in note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# L1 — 90m worklog + summary +# --------------------------------------------------------------------------- + + +class _L1Plane: + def __init__(self, durations: list[int], summary_ids: list[str] | None = None): + self.work_items = SimpleNamespace( + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + rows = [SimpleNamespace(work_item_id=i, duration=90) for i in (summary_ids or [])] + self.projects = SimpleNamespace(get_worklog_summary=lambda **kw: rows) + + +def test_l1_untouched_no_worklog_fails(): + async def _go(): + plane = _L1Plane([]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l1_wrong_duration_120_fails(): + async def _go(): + plane = _L1Plane([120], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("Logged 120 minutes; summary ok.")) + assert ok is False, note + assert "90" in note + + return asyncio.run(_go()) + + +def test_l1_empty_summary_with_90m_log_fails(): + """Reviewer counterexample: 90m log present but final text empty → fail.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("")) + assert ok is False, note + assert "duration" in note.lower() or "summary" in note.lower() + + return asyncio.run(_go()) + + +def test_l1_one_hundred_ninety_minutes_fails(): + """Reviewer counterexample: English 'ninety' must not satisfy numeric duration.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1( + plane, + ctx, + _run("Logged one hundred ninety minutes. Project summary looks fine."), + ) + assert ok is False, note + assert "duration" in note.lower() or "90" in note or "1.5" in note + + return asyncio.run(_go()) + + +def test_l1_logged_1_5_hours_with_total_passes(): + """Reviewer: numeric 1.5 is a valid duration token (with summary substance).""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("Logged 1.5 hours total.")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l1_ninety_minutes_of_work_fails_by_design(): + """Calibration: '90 minutes of work' FAILS by design. + + The prompt asks to report the project worklog summary (who/what has time + logged). An answer naming neither who nor what (no title, no person, no + 'summary'/'total') has not done that half of the task — even with a correct + numeric duration. + """ + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("90 minutes of work")) + assert ok is False, note + assert "summary" in note.lower() + + return asyncio.run(_go()) + + +def test_l1_90m_log_and_summary_text_passes(): + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1( + plane, + ctx, + _run("Logged 90 minutes. Worklog summary: 1 item with time logged."), + ) + assert ok is True, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# L2 — activities +# --------------------------------------------------------------------------- + + +class _L2Plane: + def __init__(self, n_activities: int): + acts = [SimpleNamespace(id=f"a{i}", verb="updated") for i in range(n_activities)] + self.work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: _Page(acts))) + + +def test_l2_untouched_empty_final_text_fails(): + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l2_contract_count_three_passes(): + """Contract line 'count: 3' truth=3 passes.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("Saw some history.\ncount: 3")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l2_contract_count_two_fails_truth_three(): + """Contract 'count: 2' truth=3 fails.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("count: 2")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l2_negative_contract_and_bare_fail_truth_three(): + """'-3' and 'count: -3' fail truth=3 (signed equality).""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok1, _ = await verify_l2(plane, ctx, _run("-3")) + ok2, _ = await verify_l2(plane, ctx, _run("count: -3")) + assert ok1 is False + assert ok2 is False + + return asyncio.run(_go()) + + +def test_l2_prose_only_without_contract_fails_by_design(): + """By design: prose without 'count: N' (or bare int) fails — format is part of the task.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("There are 3 activities and some comment phrases.")) + assert ok is False, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# L3 — release tag +# --------------------------------------------------------------------------- + + +class _L3Plane: + def __init__(self, versions: list[str]): + tags = [SimpleNamespace(id=f"t-{v}", version=v) for v in versions] + self.releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page(tags))) + + +def test_l3_untouched_no_tag_fails(): + async def _go(): + plane = _L3Plane([]) + ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l3_wrong_version_tag_fails(): + async def _go(): + plane = _L3Plane(["v0.0.1", "other-rc"]) + ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) + assert ok is False, note + assert L3_TAG_VERSION in note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# L4 — customer property values +# --------------------------------------------------------------------------- + + +class _L4Plane: + def __init__(self, *, props: list[Any], values: dict[str, list[str]]): + self.customers = SimpleNamespace( + properties=SimpleNamespace(list=lambda **kw: _Page(props)), + property_values=SimpleNamespace(list=lambda **kw: values), + ) + + +def test_l4_untouched_no_property_fails(): + async def _go(): + plane = _L4Plane(props=[], values={}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l4_right_property_wrong_value_fails(): + async def _go(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + plane = _L4Plane(props=[prop], values={"prop-1": ["Startup"]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + assert L4_PROP_VALUE in note or "Startup" in note or "lack" in note + + return asyncio.run(_go()) + + +def test_l4_industry_url_type_with_enterprise_fails(): + """Reviewer counterexample: name contains Industry, URL type, value Enterprise → fail.""" + + async def _go(): + prop = SimpleNamespace( + id="prop-url", + display_name="Industry", # substring / wrong exact name + name="industry", + property_type="URL", + ) + plane = _L4Plane(props=[prop], values={"prop-url": [L4_PROP_VALUE]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l4_exact_text_enterprise_passes(): + async def _go(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + plane = _L4Plane(props=[prop], values={"prop-1": [L4_PROP_VALUE]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is True, note + assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# L5 — attachment count +# --------------------------------------------------------------------------- + + +class _L5Plane: + def __init__(self, n: int): + rows = [SimpleNamespace(id=f"att-{i}") for i in range(n)] + self.work_items = SimpleNamespace(attachments=SimpleNamespace(list=lambda **kw: _Page(rows))) + + +def test_l5_untouched_empty_final_text_fails(): + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l5_bare_zero_passes(): + """Fallback: whole-answer bare '0' still passes truth=0.""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("0")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l5_multiline_ending_count_zero_passes(): + """Multi-line answer ending with 'count: 0' passes truth=0.""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5( + plane, + ctx, + _run("No files on this work item.\ncount: 0"), + ) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l5_prose_only_without_contract_fails_by_design(): + """By design: prose without contract line fails (format instruction is part of the task).""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("There are 0 attachments.")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l5_wrong_contract_count_fails(): + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("count: 10")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_reports_contract_int_unit(): + """Direct unit cases for the contract helper.""" + from evals.tasks import reports_contract_int + + assert reports_contract_int("count: 3", 3) is True + assert reports_contract_int("count: 2", 3) is False + assert reports_contract_int("-3", 3) is False + assert reports_contract_int("count: -3", 3) is False + assert reports_contract_int("0", 0) is True + assert reports_contract_int("Some prose only", 0) is False + assert reports_contract_int("preamble\ncount: 0\n", 0) is True + # Last contract line wins + assert reports_contract_int("count: 9\ncount: 3", 3) is True + assert reports_contract_int("count: 9\ncount: 3", 9) is False + + +# --------------------------------------------------------------------------- +# Sample of 6 existing verifiers (untouched + wrong-value) +# --------------------------------------------------------------------------- + + +class _R1Plane: + def __init__(self, state_name: str): + st = SimpleNamespace(id="st-1", name=state_name, group="started") + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("r1", R1_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="r1", name=R1_TITLE, state=st), + ) + self.states = SimpleNamespace( + list=lambda **kw: _Page( + [ + st, + SimpleNamespace(id="st-2", name="Done", group="completed"), + SimpleNamespace(id="st-3", name="Backlog", group="unstarted"), + ] + ) + ) + + +def test_existing_r1_untouched_empty_text_fails(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_r1_wrong_state_in_text_fails(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("Done")) + assert ok is False, note + + return asyncio.run(_go()) + + +class _R2Plane: + def __init__(self, count: int): + self._count = count + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item(f"u{i}", f"U{i}", priority="urgent") for i in range(count)]), + count=lambda **kw: ( + SimpleNamespace(total_count=count) if False else None + ), # unused; verify_r2 uses list path + ) + + +def test_existing_r2_wrong_count_in_text_fails(): + async def _go(): + # verify_r2 counts open urgent via SDK; text must match that count. + from evals.tasks import verify_r2 as _vr2 + + class Plane: + def __init__(self): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page( + [ + _item("1", "a", priority="urgent", state=SimpleNamespace(group="started")), + _item("2", "b", priority="urgent", state=SimpleNamespace(group="started")), + _item("3", "c", priority="urgent", state=SimpleNamespace(group="started")), + _item("4", "d", priority="urgent", state=SimpleNamespace(group="started")), + ] + ) + ) + self.states = SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) + ) + + # If verifier only checks text against live count, empty/wrong text fails. + ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) + assert ok is False, note + + return asyncio.run(_go()) + + +class _W2Plane: + def __init__(self, group: str, name: str): + st = SimpleNamespace(id="st", name=name, group=group) + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w2", W2_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="w2", state=st), + ) + self.states = SimpleNamespace(list=lambda **kw: _Page([st])) + + +def test_existing_w2_untouched_not_done_fails(): + async def _go(): + plane = _W2Plane("started", "In Progress") + ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w2_wrong_cancelled_group_fails(): + async def _go(): + plane = _W2Plane("cancelled", "Cancelled") + ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +class _W4Plane: + def __init__(self, name: str): + self.labels = SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(id=kw["label_id"], name=name), + list=lambda **kw: _Page([SimpleNamespace(id="triage-id", name=name)]), + ) + + +def test_existing_w4_untouched_still_triage_fails(): + async def _go(): + plane = _W4Plane("triage") + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w4_wrong_name_needs_review_fails(): + async def _go(): + plane = _W4Plane("needs-review") + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +class _W8Plane: + def __init__(self, durations: list[int]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w8", W8_TITLE)]), + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + + +def test_existing_w8_untouched_no_log_fails(): + async def _go(): + plane = _W8Plane([]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w8_wrong_duration_fails(): + async def _go(): + plane = _W8Plane([60]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_c2_untouched_empty_text_fails(): + async def _go(): + ctx = {"release_changelog_text": "Changelog entry one: OAuth login hardening."} + ok, note = await verify_c2(object(), ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_c2_wrong_release_name_fails(): + async def _go(): + ok, note = await verify_c2( + object(), + {"release_changelog_text": "Changelog entry one: OAuth login hardening."}, + _run("Release 9.9.9 shipped nothing useful."), + ) + assert ok is False, note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# Prompt binding hard-fail + dry-run markers +# --------------------------------------------------------------------------- + + +def test_prompt_bind_strict_empty_raises(): + from evals.tasks import TASKS_BY_ID, PromptBindError, format_task_prompt + + t = TASKS_BY_ID["I1"] + with pytest.raises(PromptBindError): + format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) + + +def test_prompt_bind_strict_exception_raises(): + from evals.tasks import PromptBindError, format_task_prompt + + def boom(_ctx): + raise RuntimeError("seed broken") + + task = { + "id": "X", + "prompt": "do {work_item_id}", + "prompt_bind": boom, + } + with pytest.raises(PromptBindError, match="prompt_bind failed"): + format_task_prompt(task, {"project_name": "P"}, strict=True) + + +def test_prompt_bind_dry_run_markers(): + from evals.tasks import TASKS_BY_ID, format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) + assert "" in text + assert "EVAL x" in text + + +def test_prompt_bind_strict_success(): + from evals.tasks import TASKS_BY_ID, format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt( + t, + {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, + strict=True, + ) + assert "uuid-abc" in text + assert "<" not in text + + +# --------------------------------------------------------------------------- +# Teardown deletes release_tag + customer_property +# --------------------------------------------------------------------------- + + +class _TeardownPlane: + def __init__(self): + self.deleted: list[tuple[str, str]] = [] + self.releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)]), + delete=lambda **kw: self.deleted.append(("release_tag", kw["tag_id"])), + ), + delete=lambda **kw: self.deleted.append(("release", kw.get("release_id"))), + ) + self.customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace( + id="prop-1", + display_name=L4_PROP_DISPLAY, + name="eval-industry", + ) + ] + ), + delete=lambda **kw: self.deleted.append(("customer_property", kw["property_id"])), + ), + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) + self.projects = SimpleNamespace(delete=lambda **kw: None) + self.workspace_work_item_types = SimpleNamespace(delete=lambda **kw: None) + self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) + + +def test_teardown_deletes_release_tag_and_customer_property(): + from evals.seed import teardown + + plane = _TeardownPlane() + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "project_name": "EVAL x", + "workspace_objects": [ + {"kind": "release_tag", "id": "tag-tracked"}, + {"kind": "customer_property", "id": "prop-tracked"}, + ], + } + teardown(plane, ctx) + kinds = {k for k, _ in plane.deleted} + assert "release_tag" in kinds + assert "customer_property" in kinds + # Tracked ids deleted + assert ("release_tag", "tag-tracked") in plane.deleted + assert ("customer_property", "prop-tracked") in plane.deleted + + +def test_preclean_removes_stale_tag_and_property(): + from evals.seed import _preclean_ws3_workspace_artifacts + + deleted: list[tuple[str, str]] = [] + + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), + delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), + ) + ) + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), + delete=lambda **kw: deleted.append(("prop", kw["property_id"])), + ) + ) + + _preclean_ws3_workspace_artifacts(Plane(), "ws") + assert ("tag", "t-old") in deleted + assert ("prop", "p-old") in deleted + + +def test_preclean_delete_failure_raises_for_infra_seed(): + """Found artifact that cannot be deleted must raise (harness → infra_seed).""" + from evals.seed import _preclean_ws3_workspace_artifacts + + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), + delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), + ) + ) + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) + ) + + with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): + _preclean_ws3_workspace_artifacts(Plane(), "ws") + + +def test_preclean_empty_list_is_silent(): + from evals.seed import _preclean_ws3_workspace_artifacts + + class Plane: + releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + customers = SimpleNamespace(properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + + _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise + + +# --------------------------------------------------------------------------- +# L2 activity-worker seed gate +# --------------------------------------------------------------------------- + + +def test_l2_activity_gate_raises_when_empty(): + """Empty activities list after comments → TaskSkipped env:no-activity-worker.""" + from types import SimpleNamespace + + from evals.seed import R5_TITLE, _gate_activity_worker + from evals.tasks import TaskSkipped + + class Plane: + work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) + + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + with pytest.raises(TaskSkipped, match="env:no-activity-worker"): + _gate_activity_worker(Plane(), "ws", ctx) + + +def test_l2_activity_gate_proceeds_when_nonempty(): + from types import SimpleNamespace + + from evals.seed import R5_TITLE, _gate_activity_worker + + class Plane: + work_items = SimpleNamespace( + activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) + ) + + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + _gate_activity_worker(Plane(), "ws", ctx) # no raise diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py new file mode 100644 index 00000000..52071947 --- /dev/null +++ b/tests/test_evals_drivers.py @@ -0,0 +1,998 @@ +"""Offline tests for eval agent drivers (no real CLI invocations).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +import time +from pathlib import Path +from typing import Any + +import pytest + +from evals.drivers import ( + KNOWN_DRIVERS, + AgentRun, + ClaudeCliDriver, + CodexCliDriver, + agent_run_to_harness_dict, + get_driver, + is_plane_mcp_tool, + normalize_claude_usage, + normalize_tool_call, + parse_claude_json_result, + parse_claude_transcript_calls, + parse_codex_jsonl_events, + run_cli_subprocess, + split_plane_and_client_calls, + strip_mcp_prefix, + write_claude_mcp_config, +) +from evals.run import classify_call, parse_args, stdio_server_env + +# --------------------------------------------------------------------------- +# Fixtures (constructed — never captured from live CLIs) +# --------------------------------------------------------------------------- + +# Mirrors real claude -p --output-format json (probed): input_tokens is uncached-only; +# mass lives in cache_* + modelUsage. +CLAUDE_JSON_RESULT = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "The work item is in Todo.", + "session_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "num_turns": 3, + "total_cost_usd": 0.291, + "stop_reason": "end_turn", + "usage": { + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "iterations": [ + { + "input_tokens": 2, + "output_tokens": 8, + "cache_read_input_tokens": 57985, + "cache_creation_input_tokens": 754, + "type": "message", + } + ], + "speed": "standard", + }, + "modelUsage": { + "claude-sonnet-4-20250514": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.291, + "contextWindow": 200000, + } + }, +} + +# JSON result that already embeds tool_calls (rare path) +CLAUDE_JSON_WITH_CALLS = { + **CLAUDE_JSON_RESULT, + "tool_calls": [ + { + "name": "ToolSearch", + "input": {"query": "work items", "max_results": 5}, + }, + { + "name": "mcp__plane__find_work_items", + "input": {"project": "EVAL deadbeef", "limit": 10}, + }, + { + "name": "mcp__plane__get_work_item", + "input": {"project_id": "p1", "work_item_id": "w1"}, + }, + ], +} + + +# Transcript rows (assistant + tool_use) — Claude project JSONL shape +def _transcript_lines(*, include_tool_search: bool = False) -> str: + content_blocks: list[dict] = [] + if include_tool_search: + content_blocks.append( + { + "type": "tool_use", + "id": "toolu_0", + "name": "ToolSearch", + "input": {"query": "select:find_work_items", "max_results": 1}, + } + ) + content_blocks.append( + { + "type": "tool_use", + "id": "toolu_1", + "name": "mcp__plane__list_work_items", + "input": {"project_id": "proj-1", "per_page": 25}, + } + ) + rows = [ + { + "type": "assistant", + "sessionId": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "message": { + "role": "assistant", + "content": content_blocks, + "usage": {"input_tokens": 100, "output_tokens": 20}, + }, + }, + { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "[]"}], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_2", + "name": "mcp__plane__get_work_item", + "input": {"project_id": "proj-1", "work_item_id": "wi-1"}, + } + ], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Done."}], + "stop_reason": "end_turn", + }, + }, + ] + return "\n".join(json.dumps(r) for r in rows) + "\n" + + +CODEX_JSONL = "\n".join( + [ + json.dumps( + { + "type": "session_meta", + "payload": {"id": "sess-codex-1", "cwd": "/tmp", "cli_version": "0.0-test"}, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__find_work_items", + "arguments": json.dumps({"project": "EVAL x", "limit": 5}), + "call_id": "call_1", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec_command", + "arguments": json.dumps({"cmd": "echo hi"}), + "call_id": "call_2", + }, + } + ), + json.dumps( + { + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": 5000, + "output_tokens": 200, + "cached_input_tokens": 1000, + "cache_write_input_tokens": 50, + "total_tokens": 5200, + } + }, + }, + } + ), + json.dumps( + { + "type": "event_msg", + "payload": {"type": "agent_message", "message": "All set."}, + } + ), + json.dumps({"type": "event_msg", "payload": {"type": "task_complete"}}), + ] +) + + +# --------------------------------------------------------------------------- +# strip / parse unit tests +# --------------------------------------------------------------------------- + + +def test_strip_mcp_prefix(): + assert strip_mcp_prefix("mcp__plane__list_work_items") == "list_work_items" + assert strip_mcp_prefix("mcp__plane-mcp-server__find_work_items") == "find_work_items" + assert strip_mcp_prefix("list_work_items") == "list_work_items" + assert strip_mcp_prefix("Bash") == "Bash" + + +def test_is_plane_mcp_tool(): + assert is_plane_mcp_tool("mcp__plane__find_work_items") + assert is_plane_mcp_tool("mcp__plane-foo__x") + assert not is_plane_mcp_tool("ToolSearch") + assert not is_plane_mcp_tool("Bash") + assert not is_plane_mcp_tool("mcp__other__tool") + assert not is_plane_mcp_tool("find_work_items") + + +def test_normalize_claude_usage_real_shape(): + """F2: uncached input_tokens=10 must not be treated as run total.""" + raw, total = normalize_claude_usage(CLAUDE_JSON_RESULT) + assert raw is not None + assert raw["input_tokens"] == 10 # uncached-only + assert total is not None + assert total["input_tokens"] == 10 + assert total["cache_read_input_tokens"] == 250433 + assert total["cache_creation_input_tokens"] == 33838 + assert total["output_tokens"] == 865 + assert total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert total["total_cost_usd"] == 0.291 + assert total["source"] == "modelUsage" + + +def test_parse_claude_json_result_usage_and_cost(): + out = parse_claude_json_result(CLAUDE_JSON_RESULT) + assert out["final_text"] == "The work item is in Todo." + assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + assert out["num_turns"] == 3 + assert out["usage"]["input_tokens"] == 10 + assert out["usage"]["total_cost_usd"] == 0.291 + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["calls"] == [] + assert out["stopped_reason"] == "end_turn" + + +def test_parse_claude_json_with_embedded_calls_splits_toolsearch(): + """F1: ToolSearch is client; only plane MCP tools remain in calls.""" + out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) + assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] + assert all(c["origin"] == "plane" for c in out["calls"]) + assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] + assert out["calls"][0]["args"]["limit"] == 10 + + +def test_parse_claude_transcript_calls(tmp_path: Path): + p = tmp_path / "sess.jsonl" + p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + tagged = parse_claude_transcript_calls(p) + plane, client = split_plane_and_client_calls(tagged) + assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in client] == ["ToolSearch"] + assert plane[0]["args"]["project_id"] == "proj-1" + + +def test_parse_codex_jsonl_events(): + out = parse_codex_jsonl_events(CODEX_JSONL) + assert out["session_id"] == "sess-codex-1" + assert out["final_text"] == "All set." + assert out["usage"]["input_tokens"] == 5000 + assert out["usage"]["cache_read_input_tokens"] == 1000 + # plane only in calls; exec_command is client machinery + tools = [c["tool"] for c in out["calls"]] + assert tools == ["find_work_items"] + assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] + assert out["stopped_reason"] == "end_turn" + + +# Live-captured codex v0.147.0 `codex exec --json` shape (exact four lines). +CODEX_V0147_JSONL = "\n".join( + [ + json.dumps( + { + "type": "thread.started", + "thread_id": "019ff6af-69df-7022-b353-322ffe1ececb", + } + ), + json.dumps({"type": "turn.started"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "PING"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 16050, + "cached_input_tokens": 15104, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "reasoning_output_tokens": 0, + }, + } + ), + ] +) + + +def test_parse_codex_jsonl_events_v0147_schema(): + """Parser fixture: exact four-line v0.147 stream (thread_id, PING, usage).""" + out = parse_codex_jsonl_events(CODEX_V0147_JSONL) + assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" + assert out["final_text"] == "PING" + assert out["usage"]["input_tokens"] == 16050 + assert out["usage"]["cache_read_input_tokens"] == 15104 + assert out["usage"]["cache_creation_input_tokens"] == 0 + assert out["usage"]["output_tokens"] == 5 + + +def test_parse_codex_jsonl_events_mixed_old_and_new_schema(): + """Single parser: new keys + legacy keys in one stream both contribute.""" + mixed = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, + } + ), + # Legacy call row still harvested + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__list_work_items", + "arguments": json.dumps({"project_id": "p"}), + }, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + }, + } + ), + ] + ) + out = parse_codex_jsonl_events(mixed) + assert out["session_id"] == "thread-new-1" + assert "Hello from new" in out["final_text"] + assert [c["tool"] for c in out["calls"]] == ["list_work_items"] + assert out["usage"]["input_tokens"] == 10 + + +def test_find_codex_rollout_exact_match_and_unmatched(tmp_path: Path, monkeypatch): + """Exact session id match only; no newest-after-ts substitution.""" + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" + sessions.mkdir(parents=True) + tid = "019ff6af-69df-7022-b353-322ffe1ececb" + # Unrelated newer session (must never be returned when looking for tid) + other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" + other.write_text( + json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", + encoding="utf-8", + ) + # Exact match via filename suffix + match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" + match.write_text( + json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", + encoding="utf-8", + ) + + monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout(tid) + assert found is not None + assert tid in found.name + # Must not return the other concurrent session + assert "other-session" not in found.name + + assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None + assert drivers_mod.find_codex_rollout(None) is None + + +def test_find_codex_rollout_session_meta_id(tmp_path: Path, monkeypatch): + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" + sessions.mkdir(parents=True) + p = sessions / "rollout-meta-only.jsonl" + p.write_text( + json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout("sess-meta-42") + assert found is not None + assert found.name == "rollout-meta-only.jsonl" + + +def test_codex_driver_notes_rollout_unmatched_when_no_file(tmp_path: Path, monkeypatch): + """When thread_id is known but no rollout file matches, note codex_rollout_unmatched.""" + from evals import drivers as drivers_mod + + # Empty sessions dir under fake home + (tmp_path / ".codex" / "sessions").mkdir(parents=True) + monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run, use_proxy=False) + run = driver.run_task( + "ping", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + # Final text still from stdout (new schema) — never from a wrong rollout + assert run.final_text == "PING" + # Unmatched note only when looking for enrichment; with final_text present + # need_rollout is false for final_text — still may note if no calls. + # v0147 fixture has no tool calls → need_rollout True → unmatched note. + assert "codex_rollout_unmatched" in run.notes + + +def test_max_turns_detection_from_num_turns(): + """When num_turns >= max_turns, driver reports hit_max_turns / max_turns stop.""" + payload = {**CLAUDE_JSON_RESULT, "num_turns": 15, "tool_calls": []} + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + driver = ClaudeCliDriver(runner=fake_run) + # Avoid looking for a real transcript for empty calls + run = driver.run_task( + "do the thing", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=15, + cwd=Path("/tmp"), + ) + assert run.hit_max_turns is True + assert run.stopped_reason == "max_turns" + assert run.usage_scope == "run" + assert run.usage is not None + assert run.usage["input_tokens"] == 10 # uncached-only from real shape + assert run.usage_total is not None + assert run.usage_total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + + +def test_claude_driver_falls_back_to_transcript(tmp_path: Path, monkeypatch): + session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], # force transcript path + "result": "from-json", + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + # Plant transcript where find_claude_transcript looks + munged = str(tmp_path.resolve()).replace("/", "-") + proj = Path.home() / ".claude" / "projects" / munged + proj.mkdir(parents=True, exist_ok=True) + transcript = proj / f"{session_id}.jsonl" + transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + monkeypatch.setenv("HOME", str(Path.home())) # keep real home for this test path + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, + ) + assert run.call_source == "transcript" + assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] + assert run.final_text == "from-json" + # cleanup planted file + transcript.unlink(missing_ok=True) + + +def test_claude_driver_writes_mcp_config_and_cmd_flags(tmp_path: Path): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + seen["cwd"] = kwargs.get("cwd") + # Return minimal valid JSON + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") + driver.run_task( + "hello", + mcp_env={ + "PLANE_API_KEY": "key", + "PLANE_WORKSPACE_SLUG": "slug", + "PLANE_BASE_URL": "https://api.example", + "PLANE_MCP_SURFACE": "v2", + "PATH": "/usr/bin", + }, + model="sonnet", + max_turns=7, + cwd=tmp_path, + system="sys", + ) + cmd = seen["cmd"] + assert cmd[0] == "claude" + assert "-p" in cmd + assert "--output-format" in cmd and "json" in cmd + assert "--mcp-config" in cmd + assert "--max-turns" in cmd and "7" in cmd + assert "--model" in cmd and "sonnet" in cmd + assert "--permission-mode" in cmd and "bypassPermissions" in cmd + assert "--strict-mcp-config" in cmd + # mcp-config path is a temp file cleaned after run — re-check via write helper + cfg = tmp_path / "mcp.json" + write_claude_mcp_config( + cfg, + command="/venv/bin/python", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "key"}, + ) + data = json.loads(cfg.read_text()) + assert "mcpServers" in data + assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] + + +def test_claude_driver_server_command_override(tmp_path: Path): + """External surfaces: --server-cmd replaces the default `-m plane_mcp stdio` launch.""" + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + # Capture the mcp.json content while it still exists (temp dir). + cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp_cfg"] = json.loads(cfg_path.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver( + runner=fake_run, + server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--v2"], + ) + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_MCP_TOOLS_VERSION": "v2"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp_cfg"]["mcpServers"]["plane"] + # Default use_proxy=True: command is the proxy; real server follows "--". + assert server["args"][:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + dash = server["args"].index("--") + assert server["args"][dash + 1 :] == ["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--v2"] + # PLANE_-prefixed env (incl. foreign selection vars) passes through to the child. + assert server["env"]["PLANE_MCP_TOOLS_VERSION"] == "v2" + + +def test_agent_run_dict_keeps_action_arg(): + run = AgentRun( + calls=[ + {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, + {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, + ], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + d = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=lambda t, o, a: "out_of_set", + ) + assert d["calls"][0]["action"] == "create" + assert "action" not in d["calls"][1] + + +def test_codex_driver_parses_fake_stdout_no_live(): + def fake_run(cmd, **kwargs): + assert cmd[0] == "codex" + assert "exec" in cmd + assert "--json" in cmd + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="gpt-test", + max_turns=5, + cwd=Path("/tmp"), + ) + assert run.experimental is True + assert run.call_source == "stream" + assert run.calls[0]["tool"] == "find_work_items" + assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] + assert run.usage is not None + assert run.usage["input_tokens"] == 5000 + assert run.final_text == "All set." + + +def test_codex_driver_refuses_live_by_default(): + driver = CodexCliDriver() # real subprocess.run + with pytest.raises(RuntimeError, match="refuses live"): + driver.run_task("x", mcp_env={}, model=None, max_turns=1) + + +def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): + """F1: ToolSearch must not inflate out_of_set or num_calls.""" + run = AgentRun( + calls=[ + normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), + ], + client_tool_calls=[ + normalize_tool_call("ToolSearch", {"query": "work items"}), + ], + final_text="done", + usage={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_cost_usd": 0.29, + "modelUsage": { + "claude-sonnet": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.29, + } + }, + }, + usage_total={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_input_tokens_including_cache": 10 + 250433 + 33838, + "total_cost_usd": 0.29, + "source": "modelUsage", + }, + stopped_reason="end_turn", + usage_scope="run", + call_source="transcript", + hit_max_turns=False, + wall_time_s=1.5, + ) + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate={"get_work_item"}, + classify=classify_call, + skip_result_tokens=True, + ) + assert out["num_calls"] == 1 + assert out["out_of_set_calls"] == 0 + assert out["calls"][0]["class"] == "optimal" + assert out["client_tool_call_count"] == 1 + assert out["client_tool_calls"][0]["tool"] == "ToolSearch" + # F2: cum_input_tokens null — not the misleading uncached-only 10 + assert out["cum_input_tokens"] is None + assert out["cum_input_tokens_reason"] + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["usage_per_iteration"] == [] + assert out["result_tokens_skipped_reason"] + + +def test_agent_run_hit_max_maps_to_hit_max_iterations(): + run = AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + hit_max_turns=True, + call_source="json", + ) + out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) + assert out["hit_max_iterations"] is True + assert out["stop_reason"] == "max_turns" + + +# --------------------------------------------------------------------------- +# Plumbing +# --------------------------------------------------------------------------- + + +def test_known_drivers(): + assert KNOWN_DRIVERS == {"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + + +def test_get_driver_sdk_is_none(): + assert get_driver("sdk") is None + assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) + assert isinstance(get_driver("codex-cli"), CodexCliDriver) + + +def test_parse_args_accepts_driver(): + a = parse_args(["--driver", "claude-cli", "--dry-run"]) + assert a.driver == "claude-cli" + b = parse_args(["--dry-run"]) + assert b.driver == "sdk" + + +def test_stdio_env_still_works_for_cli_drivers(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + env = stdio_server_env(surface="v2") + assert env["PLANE_MCP_SURFACE"] == "v2" + assert env["PLANE_API_KEY"] == "k" + assert "ANTHROPIC_API_KEY" not in env + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # exists but not owned by us + return True + + +def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path: Path): + """Timeout kills the whole process group, not just the parent (codex node→native case). + + Sticky CLI: parent spawns a grandchild in the same group that would keep + stdout open if only the parent were killed. Assert the runner returns + quickly and both PIDs are dead. + """ + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_cli.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + # Grandchild stays in the same process group (no start_new_session). + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + # Hold our stdout open forever (simulates grandchild pipe hold). + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired) as ei: + run_cli_subprocess( + [sys.executable, str(script)], + timeout=1.0, + capture_output=True, + text=True, + ) + elapsed = time.monotonic() - t0 + assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" + assert getattr(ei.value, "killed_process_group", False) is True + + # Wait briefly for reaping + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"process group members still alive: {alive}" + + +def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): + """killpg(leader_pid) works after the leader is reaped (no getpgid / no proc.kill fallback). + + Simulates: leader already gone, only grandchild remains in the process group. + """ + import signal + from types import SimpleNamespace + + from evals.drivers import _kill_process_group + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_leader.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + leader = subprocess.Popen( + [sys.executable, str(script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2: + break + time.sleep(0.02) + assert len(pids) == 2, pids + leader_pid, child_pid = pids + + # Kill ONLY the leader (not the group) — grandchild survives in the group. + os.kill(leader_pid, signal.SIGKILL) + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + pass + assert not _pid_alive(leader_pid) + assert _pid_alive(child_pid), "precondition: grandchild must still be alive" + + t0 = time.monotonic() + # Direct killpg(leader_pid) — pgid == original leader pid under start_new_session. + ok = _kill_process_group(SimpleNamespace(pid=leader_pid)) + assert ok is True + assert time.monotonic() - t0 < 3.0 + + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline and _pid_alive(child_pid): + time.sleep(0.05) + assert not _pid_alive(child_pid), "grandchild survived killpg after leader death" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass + + +def test_run_cli_subprocess_baseexception_kills_group(tmp_path: Path, monkeypatch): + """Non-TimeoutExpired exceptions mid-communicate must still kill the process group.""" + import evals.drivers as drivers_mod + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + real_comm = subprocess.Popen.communicate + calls = {"n": 0} + + def boom_communicate(self, *a, **k): + calls["n"] += 1 + if calls["n"] == 1: + # Wait until pidfile is written so we can assert both die. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: + break + time.sleep(0.02) + raise KeyboardInterrupt("injected mid-communicate") + return real_comm(self, *a, **k) + + monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_cli_subprocess( + [sys.executable, str(script)], + timeout=30.0, + capture_output=True, + text=True, + ) + assert time.monotonic() - t0 < 6.0 + + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2 + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"group survived BaseException path: {alive}" + # silence unused import lint if any + assert drivers_mod.run_cli_subprocess is run_cli_subprocess + + +def test_cli_driver_timeout_notes_process_group_kill(tmp_path: Path): + """ClaudeCliDriver timeout path records timeout_killed_process_group note.""" + script = tmp_path / "slow.py" + script.write_text( + textwrap.dedent( + """ + import time + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. + from evals.drivers import run_cli_subprocess as real_runner + + def short_timeout_runner(cmd, **kwargs): + kwargs = dict(kwargs) + kwargs["timeout"] = 0.5 + # Replace the CLI binary with our sticky sleeper + return real_runner([sys.executable, str(script)], **kwargs) + + driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) + t0 = time.monotonic() + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert time.monotonic() - t0 < 6.0 + assert run.stopped_reason == "timeout" + assert "timeout_killed_process_group" in run.notes diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py new file mode 100644 index 00000000..73163691 --- /dev/null +++ b/tests/test_evals_hardening.py @@ -0,0 +1,974 @@ +"""Offline tests for eval harness hardening (taxonomy, resume, seed retry, fingerprint, canary).""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest +from plane.errors.errors import HttpError + +from evals import report as report_mod +from evals import run as run_mod +from evals import seed as seed_mod +from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result +from evals.report import is_infra_error_row, load_rows, summarize +from evals.run import ( + is_infra_cli_stop_reason, + load_resume_skip_keys, + run_canary, + run_live, + should_skip_resume_row, +) +from evals.seed import create_project_with_identifier_retry, is_identifier_collision +from evals.tasks import battery_fingerprint, task_author + +# Pinned hash of the fixed synthetic catalog in test_battery_fingerprint_stable_and_sensitive. +# Recompute only if the serialization format of battery_fingerprint changes deliberately. +PINNED_SYNTHETIC_BATTERY = "81be78bde8c7" + + +def _data_rows(path: Path) -> list[dict]: + """Parse JSONL skipping meta / non-task lines (run.py writes a meta header).""" + out: list[dict] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row.get("row_type") == "meta" or row.get("task_id") is None: + continue + out.append(row) + return out + + +@pytest.fixture(autouse=True) +def _eval_creds(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +# --------------------------------------------------------------------------- +# Resume skip decision (pure) +# --------------------------------------------------------------------------- + + +def test_should_skip_resume_row_completed_success(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True + + +def test_should_skip_resume_row_verify_fail_without_error(): + # Completed attempt (agent ran, verify failed) — do not re-run on resume. + assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True + + +def test_should_skip_resume_row_infra_seed_retries(): + assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False + + +def test_should_skip_resume_row_infra_cli_retries(): + assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False + + +def test_should_skip_resume_row_non_null_error_retries(): + assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False + assert should_skip_resume_row({"error": "boom", "error_class": None}) is False + + +def test_load_resume_skip_keys_summary(tmp_path: Path): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "surface": "v2", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "surface": "v2", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "surface": "v2", "error": None, "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") + assert skip == {("R1", 0), ("W1", 0)} + assert n_skip == 2 + assert n_retry == 1 + + +def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path: Path): + """Historical error row whose later row succeeded must not inflate n_retry.""" + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "surface": "v2", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "surface": "v2", "error": None, "error_class": None, "success": True}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") + assert skip == {("R1", 0)} + assert n_skip == 1 + assert n_retry == 0 + + +def test_load_resume_skip_keys_surface_mismatch(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "surface": "full", "error": None}) + "\n") + with pytest.raises(SystemExit, match="surface"): + load_resume_skip_keys(p, surface="v2") + + +def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "surface": "v2", + "battery": "aaaaaaaaaaaa", + "model": "sonnet", + "driver": "claude-cli", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, surface="v2", battery="bbbbbbbbbbbb") + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="haiku") + with pytest.raises(SystemExit, match="driver"): + load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="sonnet", driver="sdk") + # Missing keys on older rows: pass (back-compat) + p2 = tmp_path / "old.jsonl" + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, surface="v2", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0) in skip + + +def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "error": None}) + + "\n" + + '{"task_id": "W1", "rep": 0, "surface": "v2", "error":\n', # truncated + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") + assert skip == {("R1", 0)} + assert n_skip == 1 + err = capsys.readouterr().err + assert "invalid JSON" in err + + +def test_load_resume_skip_keys_missing_file(tmp_path: Path): + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", surface="v2") + assert skip == set() and n_skip == 0 and n_retry == 0 + + +def test_parse_args_resume_and_canary(): + a = run_mod.parse_args(["--resume", "evals/results/x.jsonl", "--dry-run"]) + assert a.resume == "evals/results/x.jsonl" + b = run_mod.parse_args(["--canary", "--tasks", "R1"]) + assert b.canary is True + + +# --------------------------------------------------------------------------- +# Error taxonomy (seed raise → infra_seed row) +# --------------------------------------------------------------------------- + + +def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + + def boom_seed(plane, run_id, needs, ctx): + ctx["project_name"] = "EVAL deadbeef" + raise HttpError("identifier already taken", 409) + + monkeypatch.setattr(run_mod, "seed", boom_seed) + monkeypatch.setattr(run_mod, "teardown", lambda plane, ctx: None) + + task = { + "id": "T1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + rows = _data_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row["error_class"] == "infra_seed" + assert row["success"] is False + assert "HttpError" in (row["error"] or "") + assert "identifier" in (row["error"] or "").lower() + assert row["battery"] # fingerprint written + + +def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + + def ok_seed(plane, run_id, needs, ctx): + ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) + + monkeypatch.setattr(run_mod, "seed", ok_seed) + monkeypatch.setattr(run_mod, "teardown", lambda plane, ctx: None) + + class BoomDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + raise RuntimeError("claude cli failed: json_parse_failed") + + monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: BoomDriver()) + + task = { + "id": "T2", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (False, "nope"), + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert "RuntimeError" in (row["error"] or "") + + +def test_run_live_timeout_agent_is_infra_cli(tmp_path: Path, monkeypatch): + """Driver returns stopped_reason=timeout → row error_class=infra_cli, battery continues.""" + from evals.drivers import agent_run_to_harness_dict + + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + class TimeoutDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="timeout", + notes=["timeout after 900s"], + ) + + monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: TimeoutDriver()) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return True, "should not run" + + task = { + "id": "T3", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed + assert row["stop_reason"] == "timeout" + assert verify_calls == [] + d = agent_run_to_harness_dict( + AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout"), + optimal=set(), + alternate=set(), + classify=lambda t, o, a: "out_of_set", + ) + assert d["stop_reason"] == "timeout" + + +def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatch): + """exit 1 + parseable JSON subtype error_during_execution → infra_cli; verify not called.""" + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "MCP server crashed", + "session_id": "sess-err", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") + + monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return False, "nope" + + task = { + "id": "T4", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["stop_reason"] == "error_during_execution" + assert verify_calls == [] + assert "claude_exit=1" in (row.get("driver_notes") or []) + + +def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): + """exit 1 + subtype error_max_turns stays in the task denominator (not infra_cli).""" + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_max_turns", + "is_error": True, + "result": "hit max turns", + "session_id": "sess-max", + "num_turns": 15, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") + + monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return False, "agent exhausted turns" + + task = { + "id": "T5", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] is None + assert row["stop_reason"] == "error_max_turns" + assert row["success"] is False + assert verify_calls == [1] + + +def test_is_infra_cli_stop_reason_matrix(): + assert is_infra_cli_stop_reason("timeout") is True + assert is_infra_cli_stop_reason("error_during_execution") is True + assert is_infra_cli_stop_reason("error") is True + assert is_infra_cli_stop_reason("error_max_turns") is False + assert is_infra_cli_stop_reason("end_turn") is False + assert is_infra_cli_stop_reason("max_turns") is False + + +def test_parse_claude_json_preserves_error_subtype(): + out = parse_claude_json_result( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "x", + "session_id": "s", + "num_turns": 1, + } + ) + assert out["stopped_reason"] == "error_during_execution" + + +# --------------------------------------------------------------------------- +# Driver timeout containment +# --------------------------------------------------------------------------- + + +def test_claude_driver_timeout_returns_agent_run_not_raise(): + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=2, + cwd=Path("/tmp"), + ) + assert run.stopped_reason == "timeout" + assert run.calls == [] + assert any("timeout after" in n for n in run.notes) + + +def test_claude_driver_json_parse_failure_raises_for_infra_cli(): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") + + driver = ClaudeCliDriver(runner=fake_run) + with pytest.raises(RuntimeError, match="claude cli failed"): + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=Path("/tmp"), + ) + + +# --------------------------------------------------------------------------- +# Seed identifier retry +# --------------------------------------------------------------------------- + + +def test_create_project_retries_409_then_succeeds(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + ident = data.identifier + attempts.append(ident) + if len(attempts) < 3: + raise HttpError("Project identifier already taken", 409) + return MagicMock(id="proj-ok", identifier=ident) + + plane = MagicMock() + plane.projects = FakeProjects() + + # Force deterministic retries after first collision. + suffixes = iter(["AAAA", "BBBB"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + project = create_project_with_identifier_retry( + plane, + "ws", + name="EVAL abcd", + identifier_prefix="EV", + initial_suffix="DEAD", + ) + assert project.id == "proj-ok" + assert attempts[0] == "EVDEAD" + assert len(attempts) == 3 + assert attempts[1] != attempts[0] + assert attempts[2] != attempts[1] + assert attempts[1] == "EVAAAA" + assert attempts[2] == "EVBBBB" + + +def test_create_project_raises_after_max_409s(monkeypatch): + attempts: list[str] = [] + + class Always409: + def create(self, *, workspace_slug, data): + attempts.append(data.identifier) + raise HttpError("identifier already taken", 409) + + plane = MagicMock() + plane.projects = Always409() + suffixes = iter(["1111", "2222", "3333", "should-not-use"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 409 + assert len(attempts) == 3 + assert attempts[0] == "EV0000" + assert attempts[1] != attempts[0] + assert attempts[1] == "EV1111" + assert attempts[2] == "EV2222" + + +def test_create_project_non_collision_error_does_not_retry(): + class Fail500: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + plane = MagicMock() + plane.projects = Fail500() + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 500 + + +def test_identifier_collision_requires_status_and_language(): + assert is_identifier_collision(HttpError("identifier already taken", 409)) is True + assert is_identifier_collision(HttpError("project exists", 400)) is True + # Validation-shaped: mentions identifier but not collision language → no retry + assert is_identifier_collision(HttpError("identifier is required", 400)) is False + assert is_identifier_collision(HttpError("identifier already taken", 500)) is False + + +# --------------------------------------------------------------------------- +# Battery fingerprint + author +# --------------------------------------------------------------------------- + + +def test_task_author_default(): + assert task_author({}) == "claude" + assert task_author({"author": "alice"}) == "alice" + + +def test_battery_fingerprint_stable_and_sensitive(): + t1 = { + "id": "A", + "prompt": "p1 {project}", + "optimal_tools": {"b", "a"}, + "alternate_tools": {"c"}, + "optimal_calls": 2, + "surface_tools": { + "v2": { + "optimal_tools": {"find_work_items"}, + "alternate_tools": set(), + } + }, + } + t2 = { + "id": "B", + "prompt": "p2", + "optimal_tools": {"x"}, + "alternate_tools": set(), + "optimal_calls": 1, + "surface_tools": {}, + } + # Order of list must not matter (sorted by id). + h1 = battery_fingerprint([t2, t1]) + h2 = battery_fingerprint([t1, t2]) + assert h1 == h2 == PINNED_SYNTHETIC_BATTERY + assert len(h1) == 12 + + t1_edit = {**t1, "prompt": "p1 edited {project}"} + assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY + + # Subset of selected tasks → different fingerprint (documented ceiling). + assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY + + +def test_battery_fingerprint_catalog_is_nonempty(): + from evals.tasks import TASKS + + fp = battery_fingerprint() + assert len(fp) == 12 + assert battery_fingerprint(list(TASKS)) == fp + + +def test_battery_fingerprint_changes_with_new_debias_tasks(): + """Adding I/L content must change the catalog fingerprint (content hash).""" + from evals.tasks import TASKS, TASKS_BY_ID + + full = battery_fingerprint() + without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] + assert without_debias, "pre-debias catalog should be non-empty" + reduced = battery_fingerprint(without_debias) + assert reduced != full + # Single new task also moves the hash relative to a reduced set. + assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced + + +# --------------------------------------------------------------------------- +# Report excludes infra_ rows +# --------------------------------------------------------------------------- + + +def test_summarize_excludes_infra_errors_from_success(): + rows = [ + {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "HttpError: 409", + "error_class": "infra_seed", + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "timeout after 120s", + "error_class": "infra_cli", + }, + {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, + ] + summary = summarize(rows) + assert summary["_meta"]["infra_errors"] == 2 + assert summary["R1"]["n"] == 2 # only non-infra, non-error rows + assert summary["R1"]["k"] == 1 + assert summary["R1"]["success"] == "1/2" + assert summary["R1"]["infra_err"] == 2 + assert is_infra_error_row(rows[1]) is True + assert is_infra_error_row(rows[0]) is False + + +def test_print_table_shows_infra_errors(capsys): + summary = { + "R1": { + "n": 1, + "k": 1, + "success": "1/1", + "wilson_lo": 0.2, + "wilson_hi": 1.0, + "med_calls": 1.0, + "calls_q1": 1.0, + "calls_q3": 1.0, + "optimal_calls": 1, + "mispick_rate": 0.0, + "errored_calls": 0, + "capped": 0, + "harness_err": 0, + "infra_err": 2, + "med_result_tokens": None, + "p95_result_tokens": None, + "med_cum_input": 0.0, + }, + "_meta": {"infra_errors": 2}, + } + report_mod.print_table(summary, "Summary: test") + out = capsys.readouterr().out + assert "infra errors: 2" in out + assert "i_err" in out + assert "R1" in out + # per-task infra_err value rendered next to h_err + assert " 2" in out # i_err column value + + +def test_is_infra_error_row_covers_sdk(): + assert is_infra_error_row({"error_class": "infra_sdk"}) is True + assert is_infra_error_row({"error_class": "infra_cli"}) is True + assert is_infra_error_row({"error_class": "task"}) is False + + +def test_load_rows_dedupe_latest_wins(tmp_path: Path): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "surface": "v2", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "surface": "v2", "success": False, "num_calls": 9}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p) # default dedupe=latest + assert len(loaded) == 1 + assert loaded[0]["num_calls"] == 9 + assert loaded[0]["success"] is False + + +def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "surface": "v2", "success": True}, + {"task_id": "R1", "rep": 0, "surface": "v2", "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p, dedupe="none") + assert len(loaded) == 2 + err = capsys.readouterr().err + assert "duplicate" in err + assert "R1" in err + + +# --------------------------------------------------------------------------- +# Canary mode +# --------------------------------------------------------------------------- + + +def test_canary_detects_broken_verifier(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + async def always_ok(plane, ctx, run): + return True, "false positive" + + async def correctly_fails(plane, ctx, run): + return False, "empty agent correctly rejected" + + tasks = [ + { + "id": "GOOD", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": correctly_fails, + }, + { + "id": "BAD", + "prompt": "y {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": always_ok, + }, + ] + rc = asyncio.run(run_canary(tasks, surface="full")) + assert rc == 1 + + +def test_canary_passes_when_all_verifiers_reject(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + async def reject(plane, ctx, run): + assert run == {"final_text": "", "calls": []} + return False, "no-op rejected" + + tasks = [ + { + "id": "G1", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": reject, + }, + ] + rc = asyncio.run(run_canary(tasks, surface="full")) + assert rc == 0 + + +def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: None) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr( + run_mod, + "resolve_surface_tool_sets", + lambda task, surface: { + "skip": "unsupported on surface", + "optimal_tools": set(), + "alternate_tools": set(), + "classification": "exact", + }, + ) + tasks = [ + { + "id": "SKIPME", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (False, "unused"), + }, + ] + rc = asyncio.run(run_canary(tasks, surface="v2")) + assert rc == 1 + + +# --------------------------------------------------------------------------- +# End-to-end resume +# --------------------------------------------------------------------------- + + +def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): + out = tmp_path / "resume.jsonl" + # Pre-write: completed R1/0 + infra R2/0 (same surface/battery/model/driver as this run). + # Battery is computed from the task list below — seed the file after we know it, + # or write rows without battery (back-compat) and only check skip/retry behavior. + prior = [ + { + "task_id": "R1", + "rep": 0, + "surface": "full", + "driver": "claude-cli", + "model": "sonnet", + "error": None, + "error_class": None, + "success": True, + }, + { + "task_id": "R2", + "rep": 0, + "surface": "full", + "driver": "claude-cli", + "model": "sonnet", + "error": "HttpError: 409", + "error_class": "infra_seed", + "success": False, + }, + ] + out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") + + fake_plane = MagicMock() + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_calls: list[str] = [] + + def ok_seed(plane, run_id, needs, ctx): + # Infer task from empty ctx; runner sets project for verify path. + ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) + seed_calls.append(run_id) + + monkeypatch.setattr(run_mod, "seed", ok_seed) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + class OkDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[{"tool": "list_work_items", "args": {}, "origin": "plane"}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: OkDriver()) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + tasks = [ + { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + }, + { + "id": "R2", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + }, + ] + + rc = asyncio.run( + run_live( + tasks, + model_alias="sonnet", + reps=1, + surface="full", + out_path=out, + driver_name="claude-cli", + resume=True, + ) + ) + assert rc == 0 + # Only R2 should have been re-seeded/run (R1 completed → RESUME_SKIP). + assert len(seed_calls) == 1 + data = _data_rows(out) + # prior 2 + 1 new R2 row (meta may also exist if file was empty — it wasn't) + assert len(data) == 3 + new_r2 = data[-1] + assert new_r2["task_id"] == "R2" + assert new_r2["success"] is True + assert new_r2["error_class"] is None diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py new file mode 100644 index 00000000..1f7a05dc --- /dev/null +++ b/tests/test_evals_proxy.py @@ -0,0 +1,1795 @@ +"""Offline tests for the MCP recording proxy and proxy-first drivers.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from evals.drivers import ( + KNOWN_DRIVERS, + AntigravityCliDriver, + ClaudeCliDriver, + CodexCliDriver, + OpencodeCliDriver, + agent_run_to_harness_dict, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + get_driver, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + prepare_antigravity_fake_home, + proxy_wrap_server_command, + wait_for_proxy_meta, + write_antigravity_mcp_config, + write_opencode_mcp_config, +) +from evals.proxy import ( + SHUTDOWN_DEADLINE_S, + SidecarRecorder, + map_child_returncode, + process_buffer_lines, + reap_timeout, + scrub_child_pythonpath, + write_all_fd, +) +from evals.proxy import main as proxy_main +from evals.run import resolve_model_for_driver + +REPO = Path(__file__).resolve().parent.parent + + +@pytest.fixture(autouse=True) +def _no_redis(monkeypatch): + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +# --------------------------------------------------------------------------- +# Fake MCP server script (real subprocess, no network) +# --------------------------------------------------------------------------- + +FAKE_SERVER = textwrap.dedent( + r""" + import json, sys + + def send(obj): + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + for raw in sys.stdin: + raw = raw.strip() + if not raw: + continue + try: + msg = json.loads(raw) + except json.JSONDecodeError: + sys.stdout.write(raw + "\n") + sys.stdout.flush() + continue + mid = msg.get("id") + method = msg.get("method") + if method == "initialize": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "serverInfo": {"name": "fake"}, + }, + }) + elif method == "tools/list": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": {"tools": [{"name": "list_work_items", "inputSchema": {}}]}, + }) + elif method == "tools/call": + params = msg.get("params") or {} + name = params.get("name") + args = params.get("arguments") or {} + if name == "boom": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "content": [{"type": "text", "text": "fail"}], + "isError": True, + }, + }) + else: + body = f"ok:{name}:{json.dumps(args, sort_keys=True)}" + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "content": [{"type": "text", "text": body}], + "isError": False, + }, + }) + elif method and mid is not None: + send({"jsonrpc": "2.0", "id": mid, "result": {}}) + sys.exit(7) + """ +).lstrip() + + +def _write_fake_server(path: Path) -> Path: + path.write_text(FAKE_SERVER, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# Proxy round-trip (real subprocess) +# --------------------------------------------------------------------------- + + +def test_proxy_records_tools_call_and_exit_code(tmp_path: Path): + server = _write_fake_server(tmp_path / "fake_server.py") + sidecar = tmp_path / "side.jsonl" + cmd = [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ] + # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. + client_in = ( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {"project": "P"}}, + } + ) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "boom", "arguments": {}}, + } + ) + + "\n" + + "NOT_JSON_LINE\n" + ) + proc = subprocess.run( + cmd, + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7 # child exit propagated + # Byte-faithful: unparsed line and JSON responses appear on stdout. + out = proc.stdout.decode("utf-8", errors="replace") + assert "NOT_JSON_LINE" in out + assert "list_work_items" in out or "ok:list_work_items" in out + + rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert len(call_rows) == 2 + assert call_rows[0]["tool"] == "list_work_items" + assert call_rows[0]["args"] == {"project": "P"} + assert call_rows[0]["is_error"] is False + assert call_rows[0]["result_chars"] > 0 + assert call_rows[0]["seq"] == 1 + assert call_rows[1]["tool"] == "boom" + assert call_rows[1]["is_error"] is True + assert meta["unparsed_lines"] >= 1 + assert meta["relayed_lines"] >= 3 + + +def test_proxy_byte_faithful_child_receives_exact_bytes(tmp_path: Path): + """Child sees the exact request bytes the client sent (no re-serialization).""" + received = tmp_path / "received.bin" + echo_server = tmp_path / "echo_server.py" + echo_server.write_text( + textwrap.dedent( + f""" + import sys + data = sys.stdin.buffer.read() + open({str(received)!r}, "wb").write(data) + # Still answer initialize-ish so proxy drains cleanly + for line in data.splitlines(keepends=True): + if not line.strip(): + continue + try: + import json + msg = json.loads(line) + except Exception: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + continue + if msg.get("id") is not None: + sys.stdout.buffer.write( + (json.dumps({{"jsonrpc": "2.0", "id": msg["id"], "result": {{}}}}) + "\\n").encode() + ) + sys.stdout.buffer.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "s.jsonl" + # Deliberately non-canonical JSON spacing — re-serialization would change it. + payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(echo_server), + ], + input=payload, + capture_output=True, + cwd=str(REPO), + timeout=10, + ) + assert proc.returncode == 0 + assert received.read_bytes() == payload + + +def test_sidecar_recorder_unit(tmp_path: Path): + rec = SidecarRecorder(tmp_path / "a.jsonl") + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": {"name": "t", "arguments": {"a": 1}}, + } + ) + rec.on_server_message({"jsonrpc": "2.0", "id": 9, "result": {"content": [], "isError": False}}) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") + assert len(calls) == 1 + assert calls[0]["tool"] == "t" + assert calls[0]["args"] == {"a": 1} + assert calls[0]["origin"] == "plane" + assert rec.finalized is True + + +def test_append_after_finalize_is_dropped(tmp_path: Path): + """Once write_meta seals the sidecar, further row appends no-op (meta stays last).""" + rec = SidecarRecorder(tmp_path / "fin.jsonl") + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "before", "arguments": {}}, + } + ) + rec.on_server_message({"jsonrpc": "2.0", "id": 1, "result": {"ok": True}}) + rec.write_meta() + assert rec.finalized is True + assert rec.post_finalize_appends == 0 + + # Late pump activity after meta — must not write another call row. + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "after", "arguments": {}}, + } + ) + rec.on_server_message({"jsonrpc": "2.0", "id": 2, "result": {"ok": True}}) + rec._append({"tool": "ghost", "args": {}, "seq": 99}) # noqa: SLF001 + rec.write_meta() # second meta attempt also dropped + + assert rec.post_finalize_appends >= 2 + text = (tmp_path / "fin.jsonl").read_text(encoding="utf-8") + rows = [json.loads(ln) for ln in text.splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + call_tools = [r["tool"] for r in rows if r.get("row_type") != "proxy_meta"] + assert call_tools == ["before"] + assert "after" not in call_tools + assert "ghost" not in call_tools + assert text.count("proxy_meta") == 1 + + +def test_pumps_alive_meta_classified_incomplete(tmp_path: Path): + """proxy_meta with pumps_alive=true is incomplete (same as pending_left>0).""" + p = tmp_path / "s.jsonl" + rows = [ + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": True, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status.get("pumps_alive") is True + assert len(calls) == 1 + + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + out, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in out] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n and "pumps_alive" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def test_reap_timeout_floor_when_deadline_exhausted(): + """Kill/reap waits use remaining budget with a non-zero floor.""" + past = __import__("time").monotonic() - 10.0 + assert reap_timeout(past, floor=0.1) == 0.1 + assert reap_timeout(None, floor=0.1) == 0.1 + future = __import__("time").monotonic() + 5.0 + assert reap_timeout(future, floor=0.1) >= 4.0 + + +# --------------------------------------------------------------------------- +# Driver integration: sidecar replaces CLI calls +# --------------------------------------------------------------------------- + + +def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path: Path): + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps( + { + "tool": "find_work_items", + "args": {"q": "x"}, + "is_error": False, + "result_chars": 12, + "duration_ms": 5, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, client, src = apply_proxy_sidecar( + [{"tool": "old", "args": {}, "origin": "plane"}], + [], + side, + notes, + ) + assert src == "proxy" + assert calls[0]["tool"] == "find_work_items" + assert calls[0]["duration_ms"] == 5 + assert any("calls_from_proxy" in n for n in notes) + + +def test_apply_proxy_sidecar_empty_fallback(tmp_path: Path): + side = tmp_path / "empty.jsonl" + side.write_text("", encoding="utf-8") + notes: list[str] = [] + original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] + calls, _client, src = apply_proxy_sidecar(original, [], side, notes) + assert calls is original or calls == original + assert "proxy_sidecar_empty" in notes + assert src != "proxy" or calls == original + + +def test_claude_driver_uses_proxy_in_mcp_config(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + # Leave empty sidecar (proxy not really run under fake runner). + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["command"] == "/venv/bin/python" + assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + assert "plane_mcp" in server["args"] + assert "proxy_sidecar_empty" in run.notes + + +def test_claude_driver_proxy_disabled_no_wrap(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["args"] == ["-m", "plane_mcp", "stdio"] + + +def test_agent_run_to_harness_propagates_proxy_fields(): + from evals.drivers import AgentRun + + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {"q": "a"}, + "origin": "plane", + "is_error": True, + "result_chars": 99, + "duration_ms": 42, + } + ], + final_text="x", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + usage_scope="run", + ) + d = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=lambda t, o, a: "optimal", + ) + assert d["calls"][0]["is_error"] is True + assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["duration_ms"] == 42 + assert d["errored_calls"] == 1 + + +# --------------------------------------------------------------------------- +# Antigravity / OpenCode adapters (arg construction only) +# --------------------------------------------------------------------------- + + +def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + env = kwargs.get("env") or {} + seen["env"] = env + home = env.get("HOME") + if home: + cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" + seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, + model="gemini-2.5", + max_turns=5, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "agy" + assert "-p" in seen["cmd"] + assert "--output-format" in seen["cmd"] + assert "json" in seen["cmd"] + assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] + assert "no_turn_cap" in run.notes + assert seen.get("mcp_cfg") is not None + assert "mcpServers" in seen["mcp_cfg"] + assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) + + +def test_write_antigravity_mcp_config_shape(tmp_path: Path): + p = tmp_path / "mcp_config.json" + write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) + data = json.loads(p.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + assert data["mcpServers"]["plane"]["env"]["A"] == "1" + + +def test_opencode_driver_writes_project_config(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cwd = kwargs.get("cwd") + seen["cwd"] = cwd + cfg = Path(cwd) / "opencode.json" if cwd else None + seen["opencode_cfg"] = json.loads(cfg.read_text()) if cfg and cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + driver = OpencodeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="openai/gpt-test", + max_turns=4, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "opencode" + assert "run" in seen["cmd"] + assert "--format" in seen["cmd"] and "json" in seen["cmd"] + assert "-m" in seen["cmd"] and "openai/gpt-test" in seen["cmd"] + assert "no_turn_cap" in run.notes + data = seen["opencode_cfg"] + assert data is not None + assert data["mcp"]["plane"]["type"] == "local" + assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) + + +def test_write_opencode_mcp_config_shape(tmp_path: Path): + p = tmp_path / "opencode.json" + write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) + data = json.loads(p.read_text()) + assert data["mcp"]["plane"]["command"][0] == "py" + assert data["mcp"]["plane"]["environment"]["K"] == "V" + + +def test_known_drivers_and_get_driver(): + assert "antigravity-cli" in KNOWN_DRIVERS + assert "opencode-cli" in KNOWN_DRIVERS + assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) + assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) + + +def test_proxy_wrap_server_command(): + out = proxy_wrap_server_command( + ["python", "-m", "plane_mcp", "stdio"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="/venv/bin/python", + ) + assert out[:5] == ["/venv/bin/python", "-m", "evals.proxy", "--log", "/tmp/s.jsonl"] + assert out[5] == "--" + assert out[6:] == ["python", "-m", "plane_mcp", "stdio"] + + +def test_proxy_main_requires_command(): + with pytest.raises(SystemExit): + proxy_main(["--log", "/tmp/x.jsonl"]) + + +# --------------------------------------------------------------------------- +# Review-fix coverage +# --------------------------------------------------------------------------- + + +def test_server_initiated_request_does_not_pop_pending(tmp_path: Path): + """Server message with method+id must not complete a tools/call pending slot.""" + rec = SidecarRecorder(tmp_path / "s.jsonl") + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {}}, + } + ) + # Server-initiated request reusing id=1 (roots/list style). + rec.on_server_message({"jsonrpc": "2.0", "id": 1, "method": "roots/list", "params": {}}) + assert rec.server_requests == 1 + # Real response for the tools/call should still match. + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": "ok"}], "isError": False}, + } + ) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "s.jsonl") + assert len(calls) == 1 + assert calls[0]["tool"] == "list_work_items" + assert calls[0]["is_error"] is False + + +def test_client_response_to_server_request_ignored(tmp_path: Path): + rec = SidecarRecorder(tmp_path / "s.jsonl") + # Client answers a server request — no method, has id. + rec.on_client_message({"jsonrpc": "2.0", "id": 99, "result": {"roots": []}}) + assert rec._pending == {} # noqa: SLF001 — intentional: no pending opened + rec.write_meta() + assert load_proxy_sidecar_calls(tmp_path / "s.jsonl") == [] + + +def test_map_child_returncode_signal(): + assert map_child_returncode(0) == 0 + assert map_child_returncode(1) == 1 + assert map_child_returncode(-9) == 128 + 9 + assert map_child_returncode(-15) == 128 + 15 + assert map_child_returncode(None) == 1 + + +def test_write_all_fd_loops_on_short_writes(tmp_path: Path): + """write_all_fd must loop until all bytes are written (simulate via pipe).""" + import os + + r, w = os.pipe() + payload = b"abcdefghijklmnopqrstuvwxyz" * 100 + # Write in parent; read in same process after. + write_all_fd(w, payload) + os.close(w) + got = b"" + while True: + chunk = os.read(r, 64) + if not chunk: + break + got += chunk + os.close(r) + assert got == payload + + +def test_load_proxy_sidecar_sorts_by_seq(tmp_path: Path): + p = tmp_path / "s.jsonl" + # Append in reverse response order. + rows = [ + {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, + {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls = load_proxy_sidecar_calls(p) + assert [c["tool"] for c in calls] == ["a", "b"] + + +def test_load_proxy_sidecar_torn_final_line(tmp_path: Path): + p = tmp_path / "s.jsonl" + good = { + "tool": "a", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + # Complete call row + torn final line (no proxy_meta). + p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status["torn_line"] is True + assert status["meta"] is None + assert [c["tool"] for c in calls] == ["a"] + + +def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path: Path): + p = tmp_path / "s.jsonl" + # Incomplete: one proxy call, no meta. + p.write_text( + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in calls] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def test_proxy_exits_when_child_dies_first(tmp_path: Path): + """Child exits while parent stdin is still open — proxy must not hang.""" + server = tmp_path / "die_soon.py" + server.write_text( + textwrap.dedent( + """ + import sys, time + # Emit nothing and exit quickly; leave proxy client stdin open. + time.sleep(0.15) + sys.exit(3) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + t0 = __import__("time").monotonic() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + # Keep stdin open (do not close) so the stdin pump blocks on readline; + # the proxy must still notice child death and exit. + deadline = SHUTDOWN_DEADLINE_S + 5.0 + try: + rc = proc.wait(timeout=deadline) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + pytest.fail(f"proxy hung >{deadline}s after child exit") + elapsed = __import__("time").monotonic() - t0 + # Must finish well under the hang window (not wait the full drain). + assert elapsed < deadline + # Child's exit code (3) should propagate; tolerate signal map if the + # runtime reaps oddly, but meta must still be present. + assert rc in (3, 128 + 3) or rc == 3 + assert sidecar.is_file() + text = sidecar.read_text(encoding="utf-8") + assert "proxy_meta" in text + # Prefer exact child code when available + if rc not in (3, 128 + 3): + # At least ensure we did not hang; surface stderr for diagnosis. + err = (proc.stderr.read() if proc.stderr else b"").decode() + assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.stdin: + try: + proc.stdin.close() + except Exception: + pass + + +def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path: Path): + """proxy + plane path must resolve when cwd is a temp dir (OpenCode case).""" + server = tmp_path / "echo_once.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {"content": [], "isError": False}, + }) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign_cwd" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) + # Drop any ambient PYTHONPATH pollution by putting repo first. + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "ping", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), # foreign cwd — must still import evals.proxy + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "ping" + + +def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): + def make_fake(driver_cls: type, bag: dict): + def fake_run(cmd, **kwargs): + bag["cmd"] = cmd + if driver_cls is ClaudeCliDriver and "--mcp-config" in cmd: + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is OpencodeCliDriver: + cwd = kwargs.get("cwd") + if cwd: + cfg = Path(cwd) / "opencode.json" + if cfg.is_file(): + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is AntigravityCliDriver: + env = kwargs.get("env") or {} + home = env.get("HOME") + if home: + for rel in ( + Path(".gemini") / "config" / "mcp_config.json", + Path(".gemini") / "antigravity-cli" / "mcp_config.json", + ): + p = Path(home) / rel + if p.is_file(): + bag.setdefault("cfgs", []).append(json.loads(p.read_text())) + elif driver_cls is CodexCliDriver: + bag["cmd_joined"] = " ".join(cmd) + out = ( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + if driver_cls is ClaudeCliDriver + else "{}" + ) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + return fake_run + + for Driver, bin_key in ( + (ClaudeCliDriver, "claude_bin"), + (CodexCliDriver, "codex_bin"), + (AntigravityCliDriver, "agy_bin"), + (OpencodeCliDriver, "opencode_bin"), + ): + seen: dict = {} + kwargs = { + "runner": make_fake(Driver, seen), + "use_proxy": True, + "python_bin": sys.executable, + "server_command": ["/ext/bin/foreign-mcp", "stdio", "--v2"], + } + if Driver is CodexCliDriver: + kwargs["allow_live"] = True + kwargs[bin_key] = "fake-bin" + driver = Driver(**kwargs) + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + blob = json.dumps(seen) + assert "foreign-mcp" in blob or "foreign-mcp" in seen.get("cmd_joined", "") + + +def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") + + for Driver in (AntigravityCliDriver, OpencodeCliDriver): + d = Driver(runner=fake_run, use_proxy=False, python_bin=sys.executable) + run = d.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.call_source != "proxy" + + +def test_resolve_model_for_driver_qualification(): + assert resolve_model_for_driver("claude-cli", "sonnet") == "sonnet" + assert resolve_model_for_driver("opencode-cli", "sonnet").startswith("anthropic/") + assert resolve_model_for_driver("antigravity-cli", "haiku").startswith("gemini") + # Free-form passthrough + assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" + assert resolve_model_for_driver("sdk", "sonnet") == "claude-sonnet-5" + + +def test_ensure_proxy_pythonpath_injects_repo(): + env = ensure_proxy_pythonpath({}) + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + # Idempotent + env2 = ensure_proxy_pythonpath(env) + assert env2["PYTHONPATH"].count(str(REPO)) == 1 + + +def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): + real_home = tmp_path / "real" + cli = real_home / ".gemini" / "antigravity-cli" + cli.mkdir(parents=True) + token_path = cli / "antigravity-oauth-token" + token_path.write_text("secret", encoding="utf-8") + # Snapshot real home before setup — must be byte-identical after. + before = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + + fake = tmp_path / "fake" + prepare_antigravity_fake_home( + fake, + command="python", + args=["-m", "evals.proxy", "--log", "s", "--", "x"], + env={"PLANE_API_KEY": "k"}, + real_home=real_home, + ) + p1 = fake / ".gemini" / "config" / "mcp_config.json" + p2 = fake / ".gemini" / "antigravity-cli" / "mcp_config.json" + assert p1.is_file() and p2.is_file() + fake_cli = fake / ".gemini" / "antigravity-cli" + assert fake_cli.is_dir() and not fake_cli.is_symlink() + # Auth artifact is a plain COPY — never a symlink (no write-through path). + token = fake_cli / "antigravity-oauth-token" + assert token.is_file() and not token.is_symlink() + assert token.read_text(encoding="utf-8") == "secret" + # Writing the fake token must not mutate the real one. + token.write_text("mutated", encoding="utf-8") + assert token_path.read_text(encoding="utf-8") == "secret" + # mcp_config is a real file in the fake tree, not inside real home. + assert not (cli / "mcp_config.json").exists() + data = json.loads(p1.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + # Real home byte-for-byte untouched (including oauth token). + after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + assert after == before + + +def test_process_buffer_partial_line_and_multi_line_chunk(tmp_path: Path): + """Partial line stays buffered; two lines in one chunk both process.""" + import os + + rec = SidecarRecorder(tmp_path / "s.jsonl") + r, w = os.pipe() + buf = bytearray() + # Partial line without newline — stays buffered (no tools/call pending yet). + buf.extend(b'{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"a","arguments":{}}') + process_buffer_lines(buf, forward_fd=w, recorder=rec, is_client=True, record_jsonrpc=True) + assert len(buf) > 0 and b"\n" not in buf + assert rec._pending == {} # noqa: SLF001 + + # Complete first line + second full line in one extend (multi-line prefetch). + buf.extend(b'}\n{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"b","arguments":{}}}\n') + process_buffer_lines(buf, forward_fd=w, recorder=rec, is_client=True, record_jsonrpc=True) + assert len(buf) == 0 + assert 1 in rec._pending and 2 in rec._pending # noqa: SLF001 + os.close(w) + while os.read(r, 65536): + pass + os.close(r) + + +def test_child_exit_drains_final_response(tmp_path: Path): + """Child writes a final tools/call response then exits immediately — must record it.""" + server = tmp_path / "final_then_exit.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {"content": [{"type": "text", "text": "final"}], "isError": False}, + }) + "\\n") + sys.stdout.flush() + # Exit immediately after writing final response. + sys.exit(0) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 7, + "method": "tools/call", + "params": {"name": "final_tool", "arguments": {"x": 1}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 0 + assert b"final" in proc.stdout + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "final_tool" + assert calls[0]["is_error"] is False + + +def test_scrub_child_pythonpath_removes_repo(): + import os + + env = {"PYTHONPATH": f"{REPO}{os.pathsep}/other/lib", "FOO": "1"} + scrubbed = scrub_child_pythonpath(env) + assert "/other/lib" in scrubbed["PYTHONPATH"] + assert str(REPO) not in scrubbed["PYTHONPATH"].split(os.pathsep) + # Only-repo entry drops the var entirely + only = scrub_child_pythonpath({"PYTHONPATH": str(REPO)}) + assert "PYTHONPATH" not in only + + +def test_proxy_child_env_pythonpath_clean(tmp_path: Path): + """Real MCP child must not inherit the monorepo PYTHONPATH entry.""" + server = tmp_path / "check_env.py" + server.write_text( + textwrap.dedent( + f""" + import json, os, sys + root = {str(REPO)!r} + pp = os.environ.get("PYTHONPATH", "") + parts = [p for p in pp.split(os.pathsep) if p] + bad = root in parts + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({{ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {{"content": [{{"type": "text", "text": "bad=" + str(bad)}}], "isError": False}}, + }}) + "\\n") + sys.stdout.flush() + sys.exit(0 if not bad else 9) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(__import__("os").environ)) + assert str(REPO) in env.get("PYTHONPATH", "") + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "envcheck", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + assert b"bad=False" in proc.stdout + + +def test_timeout_harvests_sidecar_calls(tmp_path: Path): + """Claude timeout path must include sidecar calls made before the timeout.""" + side_calls = [ + { + "tool": "pre_timeout", + "args": {"a": 1}, + "is_error": False, + "result_chars": 3, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + + def fake_run(cmd, **kwargs): + # Plant a complete sidecar next to the mcp config (temp dir still alive). + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + # Sidecar path is in the same temp dir as mcp.json for Claude. + # Find sidecar from proxy args in mcp config. + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + log_idx = args.index("--log") + 1 + side = Path(args[log_idx]) + side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "pre_timeout" + + +def test_timeout_harvest_waits_for_delayed_meta(tmp_path: Path): + """After CLI kill, harvest must poll until proxy_meta appears (not read early).""" + import threading + import time as time_mod + + call_row = { + "tool": "late_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + meta_row = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": False, + } + seen: dict = {"waited": False} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + # Call row first — no meta yet (simulates proxy still finalizing). + side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") + + def write_meta_later() -> None: + time_mod.sleep(0.45) + with side.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta_row) + "\n") + seen["waited"] = True + + threading.Thread(target=write_meta_later, daemon=True).start() + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + t0 = time_mod.monotonic() + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + elapsed = time_mod.monotonic() - t0 + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "late_meta_tool" + assert seen["waited"] is True + # Must have waited for the delayed meta (~0.45s), not returned instantly. + assert elapsed >= 0.4 + assert "proxy_meta_wait_timeout" not in run.notes + + +def test_wait_for_proxy_meta_unit(tmp_path: Path): + side = tmp_path / "s.jsonl" + side.write_text("", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=0.15, poll_s=0.05) is False + side.write_text(json.dumps({"row_type": "proxy_meta", "pending_left": 0}) + "\n", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=1.0, poll_s=0.05) is True + + +def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): + """If meta never arrives, harvest still returns with incomplete note.""" + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps({"tool": "only", "args": {}, "seq": 1, "is_error": False, "result_chars": 0}) + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, _client, src = harvest_proxy_after_cli_timeout([], [], side, notes, max_wait_s=0.25) + assert "proxy_meta_wait_timeout" in notes + assert len(calls) == 1 + assert src == "proxy" + assert any("incomplete" in n for n in notes) + + +def test_rapid_response_pairing(tmp_path: Path): + """Record-before-forward: fast child responses must pair with requests (no unmatched). + + Real subprocess child replies instantly; many iterations stress the race where + a response could land on the stdout pump before _pending[id] was registered. + """ + server = tmp_path / "instant_reply.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + for line in sys.stdin.buffer: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: + continue + mid = msg.get("id") + if mid is None: + continue + # Instant reply — no sleep — maximize race window. + sys.stdout.buffer.write( + (json.dumps({ + "jsonrpc": "2.0", + "id": mid, + "result": {"content": [{"type": "text", "text": "ok"}], "isError": False}, + }) + "\\n").encode() + ) + sys.stdout.buffer.flush() + """ + ), + encoding="utf-8", + ) + n = 40 + lines = [] + for i in range(1, n + 1): + lines.append( + json.dumps( + { + "jsonrpc": "2.0", + "id": i, + "method": "tools/call", + "params": {"name": f"tool_{i}", "arguments": {"i": i}}, + } + ) + ) + client_in = "\n".join(lines) + "\n" + sidecar = tmp_path / "side.jsonl" + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=30, + ) + assert proc.returncode == 0, proc.stderr.decode() + rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert rows[-1].get("row_type") == "proxy_meta" + assert len(call_rows) == n, f"paired {len(call_rows)}/{n}; meta={meta}" + assert meta.get("unmatched_responses", 0) == 0 + assert meta.get("pending_left", 0) == 0 + tools = {r["tool"] for r in call_rows} + assert tools == {f"tool_{i}" for i in range(1, n + 1)} + + +def test_meta_is_last_row_after_forced_kill(tmp_path: Path): + """After forced child kill, proxy_meta is the last sidecar row.""" + import time as time_mod + + server = tmp_path / "hang.py" + server.write_text( + textwrap.dedent( + """ + import sys, time + # Read one line (so proxy has something to record) then hang forever. + line = sys.stdin.buffer.readline() + if line: + import json + try: + msg = json.loads(line) + mid = msg.get("id") + if mid is not None: + sys.stdout.buffer.write( + (json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n").encode() + ) + sys.stdout.buffer.flush() + except Exception: + pass + while True: + time.sleep(60) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "hang_tool", "arguments": {}}, + } + ) + + "\n" + ) + # Close stdin after one request so proxy enters shutdown while child hangs + # → kill path under SHUTDOWN_DEADLINE_S. + t0 = time_mod.monotonic() + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=SHUTDOWN_DEADLINE_S + 15, + ) + elapsed = time_mod.monotonic() - t0 + assert sidecar.is_file() + rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + assert rows, "sidecar empty" + assert rows[-1].get("row_type") == "proxy_meta" + meta = rows[-1] + # Child was hung; kill path should have fired (or child reaped after kill). + assert meta.get("child_killed") is True or proc.returncode != 0 + # Wall clock bounded by deadline (+ small slack for process startup). + assert elapsed < SHUTDOWN_DEADLINE_S + 5.0 + + +def test_bounded_shutdown_wall_clock(tmp_path: Path): + """Shutdown after child death stays within SHUTDOWN_DEADLINE_S (+ small slack).""" + import time as time_mod + + server = tmp_path / "die_after_reply.py" + server.write_text( + textwrap.dedent( + """ + import json, sys, time + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", "id": msg["id"], + "result": {"content": [], "isError": False}, + }) + "\\n") + sys.stdout.flush() + sys.exit(0) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": {"name": "quick", "arguments": {}}, + } + ) + + "\n" + ) + t0 = time_mod.monotonic() + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(REPO), + timeout=SHUTDOWN_DEADLINE_S + 5, + ) + elapsed = time_mod.monotonic() - t0 + assert proc.returncode == 0 + assert elapsed < SHUTDOWN_DEADLINE_S + 2.0 + rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + + +def test_antigravity_fallback_runner_timeout_harvests(tmp_path: Path): + """TypeError fallback path's TimeoutExpired must still harvest via wait-for-meta.""" + call_row = { + "tool": "g_tool", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + } + + def fake_run(cmd, **kwargs): + run_env = kwargs.get("env") or {} + home = run_env.get("HOME") + if home: + # First attempt includes env= — plant sidecar from dual-written mcp config, + # then reject env so the driver retries without it. + for rel in ( + Path(home) / ".gemini" / "config" / "mcp_config.json", + Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + cfg = json.loads(rel.read_text()) + args = cfg["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text( + "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", + encoding="utf-8", + ) + break + raise TypeError("runner does not accept env=") + # Fallback call (no env) times out — outer except must still harvest. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "g_tool" + + +def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert str(REPO) in seen["env"].get("PYTHONPATH", "") + + +def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): + """--server-cmd must not be Claude-only.""" + from evals import run as run_mod + + captured: dict = {} + + def fake_get_driver(name, **kwargs): + captured["name"] = name + captured["kwargs"] = kwargs + + class Dummy: + def run_task(self, *a, **k): + from evals.drivers import AgentRun + + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + call_source="json", + ) + + return Dummy() + + monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + import asyncio + + async def _verify(*a, **k): + return False, "n" + + task = { + "id": "T", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": _verify, + } + rc = asyncio.run( + run_mod.run_live( + [task], + model_alias="sonnet", + reps=1, + surface="full", + out_path=tmp_path / "o.jsonl", + driver_name="opencode-cli", + server_cmd=["/bin/foreign", "stdio"], + ) + ) + assert rc == 0 + assert captured["name"] == "opencode-cli" + assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] + + +def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path: Path): + """Proxy os.setsid() detaches from the CLI process group. + + Simulate: CLI process-group leader spawns proxy as a child (same group); + proxy main() calls setsid and leaves the group; SIGKILL the CLI group; + proxy still finalizes proxy_meta within the shutdown deadline. + """ + import os + import signal + import time + + server = tmp_path / "echo_server.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: + continue + mid = msg.get("id") + if mid is not None: + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + leader_script = tmp_path / "cli_leader.py" + leader_script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + sidecar = Path({str(sidecar)!r}) + server = Path({str(server)!r}) + # Spawn proxy in our process group (no start_new_session on child). + # proxy main() will os.setsid() and detach. + proxy = subprocess.Popen( + [ + sys.executable, "-m", "evals.proxy", + "--log", str(sidecar), + "--", + sys.executable, str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # Give setsid a moment, then write a tools/call and keep stdin open briefly. + time.sleep(0.4) + if proxy.stdin: + req = ( + '{{"jsonrpc":"2.0","id":1,"method":"tools/call",' + '"params":{{"name":"t","arguments":{{}}}}}}\\n' + ) + proxy.stdin.write(req.encode()) + proxy.stdin.flush() + # Stay alive as group leader until killed by the test harness. + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Leader is a process-group leader (like run_cli_subprocess). + env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} + leader = subprocess.Popen( + [sys.executable, str(leader_script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(REPO), + env=env, + ) + try: + # Wait until proxy has started (sidecar created) and setsid likely done. + boot = time.monotonic() + 5.0 + while time.monotonic() < boot: + if sidecar.is_file(): + break + time.sleep(0.05) + time.sleep(0.5) # allow setsid + optional tools/call + # SIGKILL the CLI process group — must NOT kill the detached proxy. + try: + os.killpg(leader.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + leader.kill() + leader.wait(timeout=1.0) + + # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. + deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 + meta_seen = False + while time.monotonic() < deadline: + if sidecar.is_file(): + text = sidecar.read_text(encoding="utf-8") + if "proxy_meta" in text: + meta_seen = True + break + time.sleep(0.1) + assert meta_seen, ( + f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" + ) + rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py new file mode 100644 index 00000000..b3eb860a --- /dev/null +++ b/tests/test_evals_report_ops.py @@ -0,0 +1,525 @@ +"""Offline tests for report stats, multi-surface table, listing tokens, cleanup, meta rows.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from evals import cleanup as cleanup_mod +from evals import report as report_mod +from evals.listing import count_tool_tokens, tool_payload_model_facing, tool_payload_wire +from evals.report import ( + ab_compare, + build_multi_surface_table, + dedupe_rows_latest, + format_surface_cell, + is_meta_row, + load_rows, + render_multi_surface_table, + sign_test_pvalue, + summarize, + wilson_interval, +) +from evals.run import ( + is_meta_or_non_task_row, + load_resume_skip_keys, + make_run_meta_row, + maybe_write_run_meta, +) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + + +# --------------------------------------------------------------------------- +# Sign test + Wilson +# --------------------------------------------------------------------------- + + +def test_sign_test_all_positive_hand_computed(): + """n=5 non-zero, all positive → two-sided p = 2 * (1/32) = 0.0625.""" + deltas = [1.0, 2.0, 3.0, 0.5, 4.0] + p = sign_test_pvalue(deltas) + assert p == pytest.approx(2.0 * (1.0 / 32.0)) + assert p == pytest.approx(0.0625) + + +def test_sign_test_four_of_five_hand_computed(): + """n=5, k=4 positive → right tail (C(5,4)+C(5,5))/32 = 6/32; p=2*6/32=0.375.""" + deltas = [1.0, 1.0, 1.0, 1.0, -1.0] + p = sign_test_pvalue(deltas) + right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 + assert p == pytest.approx(2.0 * right) + assert p == pytest.approx(0.375) + + +def test_sign_test_drops_zeros_and_none_when_empty(): + assert sign_test_pvalue([0.0, 0.0]) is None + assert sign_test_pvalue([]) is None + # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 + assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) + + +def test_wilson_interval_bounds(): + lo, hi = wilson_interval(5, 10) + assert lo == pytest.approx(0.2366, abs=1e-4) + assert hi == pytest.approx(0.7634, abs=1e-4) + lo0, hi0 = wilson_interval(0, 10) + assert lo0 == 0.0 + assert hi0 == pytest.approx(0.27754, abs=1e-4) + assert wilson_interval(0, 0) == (0.0, 0.0) + + +# --------------------------------------------------------------------------- +# load_rows: meta skip + dedupe +# --------------------------------------------------------------------------- + + +def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): + p = tmp_path / "r.jsonl" + lines = [ + json.dumps( + { + "row_type": "meta", + "run_id": "abc", + "surface": "v2", + "battery": "deadbeef0001", + "model": "sonnet", + "driver": "claude-cli", + "git_sha": "x", + "ts": "t", + } + ), + json.dumps({"surface": "v2", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "success": True, "num_calls": 2}), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + rows = load_rows(p) + assert len(rows) == 1 + assert rows[0]["task_id"] == "R1" + + +def test_dedupe_rows_latest_pure(): + rows = [ + {"task_id": "R1", "rep": 0, "surface": "full", "num_calls": 1}, + {"task_id": "R1", "rep": 0, "surface": "full", "num_calls": 5}, + {"task_id": "R2", "rep": 0, "surface": "full", "num_calls": 3}, + ] + out = dedupe_rows_latest(rows) + assert len(out) == 2 + by_id = {r["task_id"]: r for r in out} + assert by_id["R1"]["num_calls"] == 5 + assert by_id["R2"]["num_calls"] == 3 + + +def test_summarize_aggregate_wilson_and_call_variance(): + rows = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + ] + s = summarize(rows) + assert s["R1"]["n"] == 3 + assert s["R1"]["k"] == 2 + assert s["R1"]["calls_min"] == 2.0 + assert s["R1"]["calls_max"] == 6.0 + assert s["R1"]["med_calls"] == 4.0 + meta = s["_meta"] + assert meta["aggregate_k"] == 3 + assert meta["aggregate_n"] == 4 + assert 0.0 <= meta["aggregate_wilson_lo"] <= meta["aggregate_wilson_hi"] <= 1.0 + + +# --------------------------------------------------------------------------- +# A/B compare +# --------------------------------------------------------------------------- + + +def test_ab_compare_paired_deltas_and_sign_test(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired + ] + cmp = ab_compare(rows_a, rows_b) + assert cmp["n_paired"] == 2 + deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} + assert deltas["R1"] == -3.0 + assert deltas["R2"] == 1.0 + assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] + assert cmp["sign_test_p"] is not None + assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 + assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 + + +# --------------------------------------------------------------------------- +# Multi-surface table +# --------------------------------------------------------------------------- + + +def _synth_row( + tid: str, + *, + success: bool = True, + num_calls: int = 2, + alt: int | None = 0, + oos: int | None = 0, + classification: str = "exact", + skipped: str | None = None, + error: str | None = None, + error_class: str | None = None, + surface: str = "v2", +) -> dict[str, Any]: + return { + "task_id": tid, + "rep": 0, + "surface": surface, + "success": success, + "num_calls": num_calls, + "alternate_calls": alt, + "out_of_set_calls": oos, + "classification": classification, + "skipped": skipped, + "error": error, + "error_class": error_class, + "calls": [], + } + + +def test_format_surface_cell_variants(): + assert format_surface_cell(None) == "—" + assert format_surface_cell(_synth_row("R1", skipped="nope")) == "skip" + assert format_surface_cell(_synth_row("R1", error="boom")) == "ERR" + assert format_surface_cell(_synth_row("R1", error_class="infra_seed", error="x")) == "ERR" + assert format_surface_cell(_synth_row("R1", success=True, num_calls=3, alt=0, oos=0)) == "✅ 3c" + assert format_surface_cell(_synth_row("R1", success=False, num_calls=4, alt=1, oos=1)) == "❌ 4c/2mp" + # external: no mispick suffix + assert format_surface_cell(_synth_row("R1", classification="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" + + +def test_multi_surface_table_snapshot_with_external(): + legacy = [ + _synth_row("R1", surface="full", num_calls=4, alt=1, oos=0), + _synth_row("R2", surface="full", success=False, num_calls=2), + ] + v2 = [ + _synth_row("R1", surface="v2", num_calls=2, alt=0, oos=0), + _synth_row("R2", surface="v2", skipped="unsupported", num_calls=0), + ] + external = [ + _synth_row("R1", surface="akhil", classification="external", alt=None, oos=None, num_calls=3), + _synth_row("R2", surface="akhil", classification="external", alt=None, oos=None, num_calls=1, success=False), + _synth_row("R3", surface="akhil", classification="external", error="timeout", error_class="infra_cli"), + ] + table = build_multi_surface_table([("full", legacy), ("v2", v2), ("akhil", external)]) + assert table["columns"] == ["full", "v2", "akhil"] + assert "R1" in table["task_ids"] and "R3" in table["task_ids"] + assert table["cells"]["R1"]["full"] == "✅ 4c/1mp" + assert table["cells"]["R1"]["v2"] == "✅ 2c" + assert table["cells"]["R1"]["akhil"] == "✅ 3c" + assert table["cells"]["R2"]["v2"] == "skip" + assert table["cells"]["R3"]["akhil"] == "ERR" + + text = render_multi_surface_table(table, markdown=False) + assert "full" in text and "v2" in text and "akhil" in text + assert "✅ 3c" in text + assert "skip" in text + assert "ERR" in text + assert "infra 1" in text + + md = render_multi_surface_table(table, markdown=True) + assert md.startswith("| task |") + assert "| R1 |" in md + assert "---" in md + assert "**agg**" in md + + # Footer: external mispicks n/a + assert table["footer"]["akhil"]["mispicks"] is None + assert table["footer"]["full"]["mispicks"] == 1 + assert table["footer"]["akhil"]["infra_errors"] == 1 + + +def test_report_main_table_cli(tmp_path: Path, capsys): + f1 = tmp_path / "a.jsonl" + f2 = tmp_path / "b.jsonl" + f1.write_text( + json.dumps(_synth_row("R1", surface="full", num_calls=2)) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps(_synth_row("R1", surface="v2", num_calls=1)) + "\n", + encoding="utf-8", + ) + rc = report_mod.main(["--table", str(f1), str(f2)]) + assert rc == 0 + out = capsys.readouterr().out + assert "full" in out and "v2" in out + assert "R1" in out + + +def test_report_main_markdown_flag(tmp_path: Path, capsys): + f1 = tmp_path / "a.jsonl" + f1.write_text(json.dumps(_synth_row("R1", surface="v2", num_calls=1)) + "\n", encoding="utf-8") + rc = report_mod.main(["--table", "--markdown", str(f1)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.startswith("| task |") + assert "| R1 |" in out + assert "---" in out + + +def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): + p = tmp_path / "d.jsonl" + rows = [ + _synth_row("R1", surface="full", num_calls=1, success=True), + {**_synth_row("R1", surface="full", num_calls=9, success=False)}, + ] + # Both rows same (task_id, rep, surface) — latest-wins would keep one. + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + rc = report_mod.main(["--no-dedupe", str(p)]) + assert rc == 0 + # With no-dedupe, both rows enter summarize → n=2 for R1. + # (dedupe default would leave n=1.) + out = capsys.readouterr().out + assert "R1" in out + assert "2/2" in out or "1/2" in out # one success of two + + +# --------------------------------------------------------------------------- +# Meta line (run.py) +# --------------------------------------------------------------------------- + + +def test_make_run_meta_row_and_write_once(tmp_path: Path): + path = tmp_path / "out.jsonl" + meta = make_run_meta_row( + run_id="rid", + surface="v2", + battery="abcd1234ef00", + model="sonnet", + driver="claude-cli", + git_sha="deadbeef", + ts="2026-01-01T00:00:00+00:00", + ) + assert meta["row_type"] == "meta" + assert is_meta_row(meta) + assert is_meta_or_non_task_row(meta) + assert maybe_write_run_meta(path, meta) is True + # Append a data row — a truncating rewrite on the second call would destroy it. + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "success": True}) + "\n") + assert maybe_write_run_meta(path, meta) is False # file non-empty + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["row_type"] == "meta" + assert json.loads(lines[1])["task_id"] == "R1" + + +def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text( + "\n".join( + [ + json.dumps( + { + "row_type": "meta", + "surface": "v2", + "battery": "bbbbbbbbbbbb", + "model": "sonnet", + "driver": "claude-cli", + } + ), + json.dumps( + { + "task_id": "R1", + "rep": 0, + "surface": "v2", + "error": None, + "error_class": None, + "success": True, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys( + p, surface="v2", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + ) + assert skip == {("R1", 0)} + assert n_skip == 1 and n_retry == 0 + + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") + + +# --------------------------------------------------------------------------- +# Listing token counts (fake tools, no network) +# --------------------------------------------------------------------------- + + +def test_count_tool_tokens_fake_list(): + class T: + def __init__(self, name, desc, inp, out=None): + self.name = name + self.description = desc + self.inputSchema = inp + self.outputSchema = out + + tools = [ + T("alpha", "short", {"type": "object"}), + T( + "beta", + "longer description here", + {"type": "object", "properties": {"x": {"type": "string"}}}, + out={"type": "object"}, + ), + ] + # Fake encode: 1 token per character (deterministic, no tiktoken needed). + encode = lambda s: list(s) # noqa: E731 + rows, total_wire, total_model = count_tool_tokens(tools, encode=encode) + assert len(rows) == 2 + assert total_wire == sum(r.wire_tokens for r in rows) + assert total_model == sum(r.model_facing_tokens for r in rows) + # Tool with outputSchema has wire > model-facing. + beta = next(r for r in rows if r.name == "beta") + assert beta.has_output_schema is True + assert beta.wire_tokens > beta.model_facing_tokens + alpha = next(r for r in rows if r.name == "alpha") + assert alpha.has_output_schema is False + assert alpha.wire_tokens == alpha.model_facing_tokens + # Sorted by wire desc + assert rows[0].wire_tokens >= rows[1].wire_tokens + + wire = tool_payload_wire(tools[1]) + assert "output_schema" in wire + model = tool_payload_model_facing(tools[1]) + assert "output_schema" not in model + + +# --------------------------------------------------------------------------- +# Cleanup dry-run never deletes +# --------------------------------------------------------------------------- + + +def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): + projects = [ + SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), + SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), + SimpleNamespace(id="p3", name="Production", identifier="PROD"), + ] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + rc = cleanup_mod.main([]) # dry-run + assert rc == 0 + assert delete_calls == [] + out = capsys.readouterr().out + assert "EVAL deadbeef" in out + assert "dry-run" in out + assert "Production" not in out # prefix filter + + +def test_cleanup_yes_deletes(monkeypatch, capsys): + projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + rc = cleanup_mod.main(["--yes"]) + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0]["project_id"] == "p1" + + +def test_list_projects_with_prefix_filters(): + projects = [ + SimpleNamespace(id="1", name="EVAL a"), + SimpleNamespace(id="2", name="Other"), + SimpleNamespace(id="3", name="EVAL b"), + SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " + ] + calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + calls.append({"workspace_slug": workspace_slug, "params": params}) + assert params is not None + assert params.per_page == 100 + # SDK always populates next_cursor even on last page. + return SimpleNamespace( + results=projects, + next_page_results=False, + next_cursor="100:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "3"] + assert len(calls) == 1 # one page only — no infinite loop on next_cursor + assert calls[0]["params"].cursor is None + + +def test_list_projects_two_page_pagination(): + page1 = [SimpleNamespace(id="1", name="EVAL one")] + page2 = [SimpleNamespace(id="2", name="EVAL two")] + seen_cursors: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + seen_cursors.append(getattr(params, "cursor", None)) + if params.cursor is None: + return SimpleNamespace( + results=page1, + next_page_results=True, + next_cursor="100:0:0", + ) + assert params.cursor == "100:0:0" + return SimpleNamespace( + results=page2, + next_page_results=False, + next_cursor="200:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "2"] + assert seen_cursors == [None, "100:0:0"] diff --git a/tests/test_evals_surface.py b/tests/test_evals_surface.py new file mode 100644 index 00000000..f44f34fe --- /dev/null +++ b/tests/test_evals_surface.py @@ -0,0 +1,165 @@ +"""Offline tests for eval --surface plumbing and classification overlays.""" + +from __future__ import annotations + +import pytest + +from evals.run import KNOWN_SURFACES, classify_call, parse_args, stdio_server_env +from evals.tasks import TASKS_BY_ID, resolve_surface_tool_sets + + +@pytest.fixture(autouse=True) +def _eval_creds(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +def test_known_surfaces(): + assert KNOWN_SURFACES == {"full", "v2", "v2-schema"} + + +def test_stdio_env_full_does_not_set_surface(monkeypatch): + env = stdio_server_env(surface="full") + assert "PLANE_MCP_SURFACE" not in env + assert env["PLANE_API_KEY"] == "test-key" + assert env["PLANE_WORKSPACE_SLUG"] == "test-ws" + assert env["PLANE_BASE_URL"] == "https://api.plane.so" + # Never inherits ambient secrets + monkeypatch.setenv("SOME_SECRET", "x") + env2 = stdio_server_env(surface="full") + assert "SOME_SECRET" not in env2 + + +def test_stdio_env_v2_sets_plane_mcp_surface(): + env = stdio_server_env(surface="v2") + assert env["PLANE_MCP_SURFACE"] == "v2" + assert env["PLANE_API_KEY"] == "test-key" + + +def test_stdio_env_v2_schema_sets_plane_mcp_surface(): + env = stdio_server_env(surface="v2-schema") + assert env["PLANE_MCP_SURFACE"] == "v2-schema" + + +def test_parse_args_accepts_v2_and_full(): + a = parse_args(["--surface", "v2", "--dry-run"]) + assert a.surface == "v2" + b = parse_args(["--surface", "full"]) + assert b.surface == "full" + + +def test_r1_v2_overlay_exact_find_work_items(): + r1 = TASKS_BY_ID["R1"] + full = resolve_surface_tool_sets(r1, "full") + assert full["classification"] == "exact" + assert full["skip"] is None + assert "list_work_items" in full["optimal_tools"] + + v2 = resolve_surface_tool_sets(r1, "v2") + assert v2["classification"] == "exact" + assert v2["skip"] is None + assert v2["optimal_tools"] == {"find_work_items"} + assert "list_work_items" not in v2["optimal_tools"] + + +def test_w1_v2_overlay(): + w1 = TASKS_BY_ID["W1"] + v2 = resolve_surface_tool_sets(w1, "v2") + assert v2["classification"] == "exact" + assert "create_work_item" in v2["optimal_tools"] + assert "get_workspace_context" in v2["optimal_tools"] + assert v2["optimal_calls"] == 2 + + +def test_s1_v2_unsupported_skip(): + s1 = TASKS_BY_ID["S1"] + v2 = resolve_surface_tool_sets(s1, "v2") + assert v2["skip"] is not None + assert "schema" in v2["skip"].lower() or "property" in v2["skip"].lower() or "not on the" in v2["skip"] + assert v2["classification"] == "exact" + + full = resolve_surface_tool_sets(s1, "full") + assert full["skip"] is None + assert "create_work_item_property" in full["optimal_tools"] + + +def test_s1_v2_schema_overlay(): + s1 = TASKS_BY_ID["S1"] + out = resolve_surface_tool_sets(s1, "v2-schema") + assert out["skip"] is None + assert out["classification"] == "exact" + assert out["optimal_tools"] == {"resolve_work_item_type", "create_work_item_property"} + assert out["optimal_calls"] == 2 + + +def test_unknown_surface_without_overlay_is_approximate(): + """A surface with no overlay falls back to flat sets + approximate.""" + r1 = TASKS_BY_ID["R1"] + # Fabricate: use a surface name that has no overlay + out = resolve_surface_tool_sets(r1, "experimental") + assert out["classification"] == "approximate" + assert out["skip"] is None + assert out["optimal_tools"] == set(r1["optimal_tools"]) + + +def test_classify_uses_resolved_sets(): + v2 = resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2") + assert classify_call("find_work_items", v2["optimal_tools"], v2["alternate_tools"]) == "optimal" + assert classify_call("list_work_items", v2["optimal_tools"], v2["alternate_tools"]) == "out_of_set" + assert classify_call("get_work_item", v2["optimal_tools"], v2["alternate_tools"]) == "alternate" + + +def test_skip_path_no_network(monkeypatch): + """Unsupported surface skip must not call seed/teardown/agent.""" + from evals import run as run_mod + + seeded = [] + torn = [] + + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr( + run_mod, + "seed", + lambda *a, **k: seeded.append(1) or (_ for _ in ()).throw(AssertionError("seed should not run")), + ) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: torn.append(1)) + + import asyncio + import tempfile + from pathlib import Path + + # Avoid importing anthropic for skip-only path: patch AsyncAnthropic too + class _FakeAnthro: + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return None + + monkeypatch.setattr("anthropic.AsyncAnthropic", lambda: _FakeAnthro()) + + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "out.jsonl" + rc = asyncio.run( + run_mod.run_live( + [TASKS_BY_ID["S1"]], + model_alias="sonnet", + reps=1, + surface="v2", + out_path=out, + ) + ) + assert rc == 0 + assert seeded == [] + text = out.read_text(encoding="utf-8") + assert "S1" in text + assert "skipped" in text + # First line may be meta header; pick the task row. + rows = [__import__("json").loads(ln) for ln in text.strip().splitlines() if ln.strip()] + row = next(r for r in rows if r.get("task_id") == "S1") + assert row["surface"] == "v2" + assert row["skipped"] + assert row["classification"] == "exact" diff --git a/tests/test_evals_verifiers.py b/tests/test_evals_verifiers.py new file mode 100644 index 00000000..bc776d50 --- /dev/null +++ b/tests/test_evals_verifiers.py @@ -0,0 +1,731 @@ +"""Adversarial offline tests for eval verifiers (false-PASS / false-FAIL regressions). + +Each test constructs a minimal fake plane client + ctx that embodies the bug +scenario and asserts the fixed verifier now returns the correct outcome. +No network; no live Plane. +""" + +from __future__ import annotations + +import asyncio +from datetime import date, timedelta +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.seed import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + CYCLE_PAST, + R1_TITLE, + W3_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, + W7_URL, + W8_TITLE, +) +from evals.tasks import ( + verify_c1, + verify_r3, + verify_s3, + verify_s5, + verify_w3, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w8, +) + +# --------------------------------------------------------------------------- +# Tiny helpers +# --------------------------------------------------------------------------- + + +class _Page: + def __init__(self, results: list[Any] | None = None, next_page_results: bool = False): + self.results = results or [] + self.next_page_results = next_page_results + self.next_cursor = None + + +def _http404() -> HttpError: + return HttpError("not found", status_code=404, response={}) + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +def _run() -> dict[str, Any]: + return {"final_text": "", "calls": []} + + +@pytest.fixture(autouse=True) +def _no_redis(monkeypatch): + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +# --------------------------------------------------------------------------- +# F1 W7 — reverse blocked_by must NOT pass +# --------------------------------------------------------------------------- + + +class _DepsDump: + def __init__(self, data: dict): + self._data = data + + def model_dump(self) -> dict: + return self._data + + +class _W7Plane: + """Fake where dependencies.list returns dump with tgt only in blocked_by.""" + + def __init__(self, deps_dump: dict, urls: list[str] | None = None): + self._deps_dump = deps_dump + self._urls = urls if urls is not None else [W7_URL] + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("src-1", W7_SOURCE_TITLE), _item("tgt-1", W7_TARGET_TITLE)]), + dependencies=SimpleNamespace(list=self._deps_list), + links=SimpleNamespace(list=self._links_list), + ) + + def _deps_list(self, **kw): + return _DepsDump(self._deps_dump) + + def _links_list(self, **kw): + return _Page([SimpleNamespace(url=u) for u in self._urls]) + + +def test_f1_w7_blocked_by_only_does_not_pass(): + async def _go(): + """tgt id only in blocked_by (reverse) must FAIL — not pass via str(dump).""" + plane = _W7Plane( + { + "blocking": [], + "blocked_by": [{"id": "tgt-1"}], # reverse direction only + } + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w7(plane, ctx, _run()) + assert ok is False, note + assert "blocking" in note.lower() or "no blocking" in note.lower() + assert "wrong direction" in note or "tgt-1" in note + + return asyncio.run(_go()) + + +def test_f1_w7_blocking_passes(): + async def _go(): + plane = _W7Plane({"blocking": [{"id": "tgt-1"}], "blocked_by": []}) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w7(plane, ctx, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F2 W6 — seeded past end_date alone must NOT pass as closed + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _W6Plane: + def __init__(self, *, past_end: str, archived_at=None, snapshot=None, sprint13_names: list[str] | None = None): + self._past_end = past_end + self._archived_at = archived_at + self._snapshot = snapshot + self._s13 = sprint13_names or [] + self.cycles = SimpleNamespace(retrieve=self._retrieve, list_work_items=self._list_wi) + + def _retrieve(self, **kw): + return SimpleNamespace( + id="c12", + end_date=self._past_end, + archived_at=self._archived_at, + progress_snapshot=self._snapshot, + ) + + def _list_wi(self, **kw): + return _Page([_item(f"i{i}", n) for i, n in enumerate(self._s13)]) + + +def test_f2_w6_seeded_past_end_alone_fails(): + async def _go(): + """Sprint 12 still at seed end_date (today-14) with no archive → not closed.""" + seed_end = (date.today() - timedelta(days=14)).isoformat() + plane = _W6Plane(past_end=seed_end, sprint13_names=["Inventory count goes negative under load"]) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": seed_end, + "w6_unfinished_titles": ["Inventory count goes negative under load"], + "cycles": {CYCLE_PAST: "c12"}, + } + ok, note = await verify_w6(plane, ctx, _run()) + assert ok is False, note + assert "not closed" in note.lower() or "Sprint 12 not closed" in note + + return asyncio.run(_go()) + + +def test_f2_w6_end_date_today_as_timestamp_passes_close(): + async def _go(): + """The API returns end_date as a timestamp, not a bare date. + + Sprint 12 closed today comes back as 'T00:00:00Z'; comparing the + whole string to today's date never matches, which silently killed the + complete_cycle close signal and left the transfer side effect + (progress_snapshot) as the only way to pass. + """ + today = date.today().isoformat() + titles = ["Inventory count goes negative under load"] + plane = _W6Plane(past_end=f"{today}T00:00:00Z", sprint13_names=titles) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() + timedelta(days=1)).isoformat(), + "w6_unfinished_titles": titles, + } + ok, note = await verify_w6(plane, ctx, _run()) + assert ok is True, note + assert "end_date" in note + + return asyncio.run(_go()) + + +def test_f2_w6_open_seed_end_tomorrow_alone_fails(): + async def _go(): + """A no-op agent leaves Sprint 12 ending tomorrow → not closed.""" + seed_end = (date.today() + timedelta(days=1)).isoformat() + titles = ["Inventory count goes negative under load"] + plane = _W6Plane(past_end=f"{seed_end}T00:00:00Z", sprint13_names=titles) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": seed_end, + "w6_unfinished_titles": titles, + } + ok, note = await verify_w6(plane, ctx, _run()) + assert ok is False, note + assert "not closed" in note.lower() + + return asyncio.run(_go()) + + +def test_f2_w6_end_date_today_passes_close(): + async def _go(): + today = date.today().isoformat() + plane = _W6Plane( + past_end=today, + sprint13_names=[ + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", + ], + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() - timedelta(days=14)).isoformat(), + "w6_unfinished_titles": [ + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", + ], + } + ok, note = await verify_w6(plane, ctx, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F3 W5 — 404 without archived list entry is NOT archive + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _W5Plane: + def __init__(self, *, retrieve_map: dict[str, Any], archived_ids: list[str]): + self._retrieve_map = retrieve_map + self._archived_ids = archived_ids + self.work_items = SimpleNamespace( + retrieve=self._retrieve, + list_archived=self._list_archived, + list=lambda **kw: _Page([]), + ) + + def _retrieve(self, **kw): + wid = str(kw["work_item_id"]) + if wid not in self._retrieve_map: + raise _http404() + val = self._retrieve_map[wid] + if val is None: + raise _http404() + return val + + def _list_archived(self, **kw): + return _Page([_item(i, f"n-{i}") for i in self._archived_ids]) + + +def test_f3_w5_deleted_404_without_archive_fails(): + async def _go(): + """All items 404 and archived list empty → fail (deletes ≠ archive).""" + ids = ["m1", "m2", "m3"] + plane = _W5Plane(retrieve_map={}, archived_ids=[]) # all 404, none archived + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "module_completed_ids": ids, + } + ok, note = await verify_w5(plane, ctx, _run()) + assert ok is False, note + assert "not archived" in note + + return asyncio.run(_go()) + + +def test_f3_w5_404_present_in_archived_passes(): + async def _go(): + ids = ["m1", "m2", "m3"] + plane = _W5Plane(retrieve_map={}, archived_ids=ids) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ids} + ok, note = await verify_w5(plane, ctx, _run()) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_f3_w5_archived_at_on_retrieve_passes(): + async def _go(): + ids = ["m1"] + plane = _W5Plane( + retrieve_map={"m1": SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z")}, + archived_ids=[], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ids} + ok, note = await verify_w5(plane, ctx, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F4 C1 — must link exact R1 item; acme* / random links fail + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _C1Plane: + def __init__( + self, + *, + customers: list[Any], + requests: list[Any], + linked: list[Any], + project_items: list[Any], + ): + self.customers = SimpleNamespace( + list=lambda **kw: _Page(customers), + requests=SimpleNamespace(list=lambda **kw: _Page(requests)), + work_items=SimpleNamespace(list=lambda **kw: _Page(linked)), + ) + self.work_items = SimpleNamespace(list=lambda **kw: _Page(project_items)) + + +def test_f4_c1_wrong_customer_name_fails(): + async def _go(): + plane = _C1Plane( + customers=[SimpleNamespace(id="c1", name="Acme Industries")], + requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], + linked=[SimpleNamespace(id="wi-r1")], + project_items=[_item("wi-r1", R1_TITLE)], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} + ok, note = await verify_c1(plane, ctx, _run()) + assert ok is False, note + assert CUSTOMER_NAME in note + + return asyncio.run(_go()) + + +def test_f4_c1_linked_other_item_not_r1_fails(): + async def _go(): + plane = _C1Plane( + customers=[SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], + linked=[SimpleNamespace(id="wi-other")], # not R1 + project_items=[_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_c1(plane, ctx, _run()) + assert ok is False, note + assert "not linked" in note or "wi-r1" in note + + return asyncio.run(_go()) + + +def test_f4_c1_exact_r1_link_passes(): + async def _go(): + plane = _C1Plane( + customers=[SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], + linked=[SimpleNamespace(id="wi-r1")], + project_items=[_item("wi-r1", R1_TITLE)], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_c1(plane, ctx, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F5 W3 — comment must contain 'contrast tokens' + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _W3Plane: + def __init__(self, comments: list[Any]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w3", W3_TITLE)]), + comments=SimpleNamespace(list=lambda **kw: _Page(comments)), + ) + + +def test_f5_w3_unrelated_comment_fails(): + async def _go(): + plane = _W3Plane([SimpleNamespace(comment_html="

lgtm

", comment_stripped="lgtm")]) + ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "contrast tokens" in note + + return asyncio.run(_go()) + + +def test_f5_w3_phrase_in_html_passes(): + async def _go(): + plane = _W3Plane( + [ + SimpleNamespace( + comment_html="

Reviewed contrast tokens — needs design pass

", + comment_stripped="Reviewed contrast tokens — needs design pass", + ) + ] + ) + ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F7 W4 — seeded triage id is authoritative + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _W4Plane: + def __init__(self, *, by_id: dict[str, Any], listed: list[Any]): + self._by_id = by_id + self.labels = SimpleNamespace(retrieve=self._retrieve, list=lambda **kw: _Page(listed)) + + def _retrieve(self, **kw): + lid = str(kw["label_id"]) + if lid not in self._by_id: + raise _http404() + return self._by_id[lid] + + +def test_f7_w4_wrong_label_renamed_triage_id_unchanged_fails(): + async def _go(): + """Name-scan would see needs-triage, but seeded triage id still named triage.""" + plane = _W4Plane( + by_id={"triage-id": SimpleNamespace(id="triage-id", name="triage")}, + listed=[ + SimpleNamespace(id="triage-id", name="triage"), + SimpleNamespace(id="other", name="needs-triage"), # wrong label renamed + ], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is False, note + assert "triage-id" in note + + return asyncio.run(_go()) + + +def test_f7_w4_seeded_id_renamed_passes(): + async def _go(): + plane = _W4Plane( + by_id={"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, + listed=[SimpleNamespace(id="triage-id", name="needs-triage")], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F6 S3 — required OPTION must not pass as TEXT + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +class _S3Plane: + def __init__(self, *, props: list[Any], types: list[Any] | None = None, workspace_owns: bool = False): + self._props = props + self._types = types if types is not None else [SimpleNamespace(id="t-inc", name="Incident")] + self._ws_owns = workspace_owns + self.work_item_types = SimpleNamespace(list=lambda **kw: self._types) + self.work_item_properties = SimpleNamespace(list=lambda **kw: self._props) + self.workspaces = SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": self._ws_owns}) + ) + self.workspace_work_item_types = SimpleNamespace(list=lambda **kw: []) + + +def test_f6_s3_required_option_does_not_pass(): + async def _go(): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name="Severity", + property_type="OPTION", + is_required=True, + ) + ] + ) + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "TEXT" in note + + return asyncio.run(_go()) + + +def test_f6_s3_required_text_passes(): + async def _go(): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name="Impact summary", + property_type="TEXT", + is_required=True, + ) + ] + ) + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F9 S3 — workspace types found via get_features probe (not seed flag) + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +def test_f9_s3_workspace_type_via_features_probe(): + async def _go(): + """Incident only on workspace types; empty project list; needs empty (no seed flag).""" + plane = _S3Plane(props=[], types=[], workspace_owns=True) + # Override workspace type list to include Incident + plane.workspace_work_item_types = SimpleNamespace( + list=lambda **kw: [SimpleNamespace(id="ws-inc", name="Incident")] + ) + plane.work_item_properties = SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace( + id="p1", + display_name="Impact summary", + property_type="TEXT", + is_required=True, + ) + ] + ) + # No bug_type_workspace_level in ctx — old code would miss Incident. + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F8 R3 due date stays in current week (seed helper) + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +def test_f8_r3_due_date_clamped_to_iso_week(): + """today+2d on Sat/Sun must not leave the week — replicate seed formula.""" + for weekday in range(7): # 0=Mon … 6=Sun + # Build a fixed "today" with that weekday relative to a known Monday. + # 2026-08-10 is a Monday. + monday = date(2026, 8, 10) + today = monday + timedelta(days=weekday) + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + # Sunday of that week + week_end = today + timedelta(days=days_to_week_end) + week_start = today - timedelta(days=today.weekday()) + assert week_start <= due <= week_end, f"weekday={weekday} due={due}" + # Specifically: Sat/Sun must not go past Sunday + if weekday >= 5: + assert due <= week_end + assert due == week_end or due == today # Sun→today, Sat→Sun + + +def test_f8_seed_r3_due_date_function_matches(): + """Import seed's computation path by re-running the formula against weekends.""" + # Saturday + sat = date(2026, 8, 15) # known Saturday + assert sat.weekday() == 5 + days_to_week_end = 6 - sat.weekday() + due = min(sat + timedelta(days=2), sat + timedelta(days=days_to_week_end)) + assert due == date(2026, 8, 16) # Sunday, not Monday 17 + # Sunday + sun = date(2026, 8, 16) + days_to_week_end = 6 - sun.weekday() + due = min(sun + timedelta(days=2), sun + timedelta(days=days_to_week_end)) + assert due == sun + + +# --------------------------------------------------------------------------- +# Minor W8 — exactly 120, not >= 120 +# --------------------------------------------------------------------------- + + +class _W8Plane: + def __init__(self, durations: list[int]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w8", W8_TITLE)]), + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + + +def test_minor_w8_480_minutes_fails(): + async def _go(): + plane = _W8Plane([480]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "120" in note + + return asyncio.run(_go()) + + +def test_minor_w8_exactly_120_passes(): + async def _go(): + plane = _W8Plane([120]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # Minor R3 — all titles required; count alone insufficient + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + +def test_minor_r3_count_without_titles_fails(): + async def _go(): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + run = {"final_text": "There are 2 items due this week.", "calls": []} + ok, note = await verify_r3( + object(), + {"r3_due_titles": titles, "r3_due_count": 2}, + run, + ) + assert ok is False, note + assert "missing title" in note.lower() or "missing title(s)" in note + + return asyncio.run(_go()) + + +# --------------------------------------------------------------------------- +# S5 — both cycle_view and is_time_tracking_enabled required +# --------------------------------------------------------------------------- + + +class _S5Plane: + def __init__( + self, + *, + cycle_view: bool, + time_tracking: bool, + customers: bool = True, + features_cycles: bool | None = None, + ): + self.projects = SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + id=kw["project_id"], + cycle_view=cycle_view, + is_time_tracking_enabled=time_tracking, + ), + get_features=lambda **kw: SimpleNamespace( + model_dump=lambda: { + "cycles": features_cycles if features_cycles is not None else cycle_view, + "modules": False, + } + ), + ) + self.workspaces = SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"customers": customers}) + ) + + +def test_s5_only_cycles_enabled_fails(): + """Two-of-three: cycles on, worklogs off, customers on → fail.""" + + async def _go(): + plane = _S5Plane(cycle_view=True, time_tracking=False, customers=True) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "is_time_tracking_enabled" in note + + return asyncio.run(_go()) + + +def test_s5_only_worklogs_enabled_fails(): + async def _go(): + plane = _S5Plane(cycle_view=False, time_tracking=True, customers=True) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "cycle_view" in note + + return asyncio.run(_go()) + + +def test_s5_workspace_customers_on_but_project_flags_off_fails(): + """Workspace customers alone is not enough — project gates must also pass.""" + + async def _go(): + plane = _S5Plane(cycle_view=False, time_tracking=False, customers=True) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "cycle_view" in note + assert "is_time_tracking_enabled" in note + + return asyncio.run(_go()) + + +def test_s5_project_flags_on_but_customers_off_fails(): + """Two-of-three: project ok, workspace customers off → fail.""" + + async def _go(): + plane = _S5Plane(cycle_view=True, time_tracking=True, customers=False, features_cycles=True) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "customers" in note + + return asyncio.run(_go()) + + +def test_s5_all_three_enabled_passes(): + async def _go(): + plane = _S5Plane(cycle_view=True, time_tracking=True, customers=True, features_cycles=True) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + return asyncio.run(_go()) From 936c072003c1440abbfb6e7e01193ca37a8cc70f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 01:04:01 +0530 Subject: [PATCH 02/93] Split the driver monolith into a package drivers.py had grown to 1947 lines holding four unrelated jobs: recording-proxy plumbing, CLI subprocess lifecycle, four vendors' transcript parsers, and the four driver classes. Each vendor's config writer, launcher and output parser sat scattered across it, so adding a driver meant touching the same file in four places and reading past three other vendors' quirks to do it. Now base/process/sidecar hold the vendor-neutral machinery and each vendor owns one module. Public imports are unchanged: evals/drivers/__init__.py re-exports the same names, built from the actual call sites rather than guessed. The boundary immediately paid for itself. Generic row mapping was reaching into Claude's usage parser whenever a CLI driver left usage_total unset, which would apply one vendor's token accounting to another's usage dict and report the result as fact. Drivers now own their own normalization and a missing total stays None, which is honest; a test pins that. Names that cross a module boundary lost their underscore prefix, since an imported name is a public name. Co-Authored-By: Claude Fable 5 --- evals/drivers.py | 1947 ---------------------------------- evals/drivers/__init__.py | 142 +++ evals/drivers/antigravity.py | 276 +++++ evals/drivers/base.py | 276 +++++ evals/drivers/claude.py | 495 +++++++++ evals/drivers/codex.py | 431 ++++++++ evals/drivers/opencode.py | 212 ++++ evals/drivers/process.py | 135 +++ evals/drivers/sidecar.py | 233 ++++ tests/test_evals_drivers.py | 43 +- 10 files changed, 2237 insertions(+), 1953 deletions(-) delete mode 100644 evals/drivers.py create mode 100644 evals/drivers/__init__.py create mode 100644 evals/drivers/antigravity.py create mode 100644 evals/drivers/base.py create mode 100644 evals/drivers/claude.py create mode 100644 evals/drivers/codex.py create mode 100644 evals/drivers/opencode.py create mode 100644 evals/drivers/process.py create mode 100644 evals/drivers/sidecar.py diff --git a/evals/drivers.py b/evals/drivers.py deleted file mode 100644 index 0de55bc6..00000000 --- a/evals/drivers.py +++ /dev/null @@ -1,1947 +0,0 @@ -"""Agent-driver abstraction for the Plane MCP eval harness. - -Drivers run one task against a tool surface and return a normalized -``AgentRun``. The default ``sdk`` driver preserves the historical Anthropic -SDK + in-process MCP client path. CLI drivers (``claude-cli``, ``codex-cli``) -spawn locally installed agent CLIs on the user's subscription — no Anthropic -API key required for those paths. - -Real CLI surfaces (probed on this machine, 2026-08-12): - -Claude Code (``claude`` v2.1.228): - - ``-p`` / ``--print`` headless - - ``--mcp-config `` (repeatable; ``--strict-mcp-config``) - - ``--output-format json|text|stream-json`` (print mode) - - ``--max-turns `` (print mode; *hidden* from ``--help`` but present) - - ``--model `` - - ``--permission-mode`` choices: acceptEdits, auto, bypassPermissions, - manual, dontAsk, plan - - ``--dangerously-skip-permissions``, ``--allowedTools`` / ``--allowed-tools`` - - Transcript: ``~/.claude/projects//.jsonl`` - with ``assistant`` rows whose ``message.content`` holds ``tool_use`` blocks. - - MCP tools surface as ``mcp____`` — strip for classification. - -Codex (``codex exec``): - - ``codex exec --json`` JSONL events on stdout - - ``-c key=value`` / ``--config`` for config.toml overrides (incl. mcp_servers) - - ``-m`` / ``--model`` - - Session rollouts: ``~/.codex/sessions/**/rollout-*.jsonl`` with - ``response_item`` / ``function_call`` payloads (name + arguments JSON string) - - Marked **experimental**; live runs are opt-in (metered quota). -""" - -from __future__ import annotations - -import json -import os -import re -import signal -import subprocess -import sys -import tempfile -import time -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Protocol - -REPO_ROOT = Path(__file__).resolve().parent.parent - -# mcp__plane__list_work_items → list_work_items -# mcp__plane-mcp-server__foo → foo -_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") -# Alternate: mcp__server__tool with multi-segment server names -_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") - - -def proxy_wrap_server_command( - real_command: list[str], - *, - sidecar_path: Path, - python_bin: str | None = None, -) -> list[str]: - """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" - py = python_bin or sys.executable - return [py, "-m", "evals.proxy", "--log", str(sidecar_path), "--", *real_command] - - -def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: - """Inject the repo root into PYTHONPATH so ``python -m evals.proxy`` works from any cwd. - - ``evals`` is not an installed package (pyproject excludes it); the MCP child - is often launched from a foreign temp dir (OpenCode project dir, etc.). - """ - root = str(REPO_ROOT) - out = dict(env) - existing = out.get("PYTHONPATH", "") - parts = [p for p in existing.split(os.pathsep) if p] - if root not in parts: - out["PYTHONPATH"] = root + (os.pathsep + existing if existing else "") - return out - - -def load_proxy_sidecar( - path: Path, -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Load sidecar call rows (sorted by seq) plus a status dict. - - Status keys: - - missing / empty / complete / incomplete - - torn_line: final line failed to parse - - meta: proxy_meta row if present - - pending_left: from meta when present - """ - status: dict[str, Any] = { - "state": "missing", - "torn_line": False, - "meta": None, - "pending_left": None, - } - if not path.is_file(): - return [], status - try: - raw = path.read_bytes() - except OSError: - return [], status - if not raw: - status["state"] = "empty" - return [], status - - # Decode with replacement so invalid UTF-8 does not crash the loader. - text = raw.decode("utf-8", errors="replace") - lines = text.splitlines() - calls: list[dict[str, Any]] = [] - meta: dict[str, Any] | None = None - torn = False - for i, line in enumerate(lines): - s = line.strip() - if not s: - continue - try: - row = json.loads(s) - except json.JSONDecodeError: - # Tolerate a torn final line (crash mid-write); stop there. - if i == len(lines) - 1: - torn = True - break - continue - if not isinstance(row, dict): - continue - if row.get("row_type") == "proxy_meta": - meta = row - continue - tool = row.get("tool") - if not tool: - continue - calls.append( - { - "tool": str(tool), - "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), - "origin": "plane", - "is_error": bool(row.get("is_error")), - "result_chars": int(row.get("result_chars") or 0), - "duration_ms": row.get("duration_ms"), - "seq": row.get("seq"), - } - ) - - # Score order must match request seq, not response-append order. - calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) - - status["torn_line"] = torn - status["meta"] = meta - if meta is not None: - status["pending_left"] = meta.get("pending_left") - status["pumps_alive"] = bool(meta.get("pumps_alive")) - incomplete = bool( - torn - or meta is None - or (meta is not None and int(meta.get("pending_left") or 0) > 0) - or (meta is not None and bool(meta.get("pumps_alive"))) - ) - if not calls and not meta and not torn: - status["state"] = "empty" - elif incomplete: - status["state"] = "incomplete" - else: - status["state"] = "complete" - return calls, status - - -def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: - """Convenience: call rows only (sorted by seq).""" - calls, _status = load_proxy_sidecar(path) - return calls - - -def apply_proxy_sidecar( - calls: list[dict[str, Any]], - client_calls: list[dict[str, Any]], - sidecar_path: Path, - notes: list[str], -) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: - """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. - - Incomplete sidecar (torn line, missing meta, pending_left>0) yields to the - CLI trace when the CLI has *more* plane calls. Returns - ``(plane_calls, client_calls, call_source)``. - """ - proxy_calls, status = load_proxy_sidecar(sidecar_path) - state = status.get("state") - if state in ("missing", "empty"): - notes.append("proxy_sidecar_empty") - return calls, client_calls, "json" - if state == "incomplete": - notes.append( - "proxy_sidecar_incomplete" - + (":torn" if status.get("torn_line") else "") - + (":no_meta" if status.get("meta") is None else "") - + (f":pending_left={status.get('pending_left')}" if status.get("pending_left") else "") - + (":pumps_alive" if status.get("pumps_alive") else "") - ) - if len(calls) > len(proxy_calls): - notes.append("proxy_sidecar_deferred_to_cli_trace") - return calls, client_calls, "json" - if proxy_calls: - notes.append(f"calls_from_proxy:{sidecar_path}") - return proxy_calls, client_calls, "proxy" - return calls, client_calls, "json" - # complete - notes.append(f"calls_from_proxy:{sidecar_path}") - return proxy_calls, client_calls, "proxy" - - -def wait_for_proxy_meta( - sidecar_path: Path, - *, - poll_s: float = 0.2, - max_wait_s: float | None = None, -) -> bool: - """Poll until the sidecar gains a ``proxy_meta`` row (or the wait expires). - - After a CLI timeout the driver kills the CLI; the proxy is a *separate* - process that only then sees stdin EOF and needs up to - ``SHUTDOWN_DEADLINE_S`` to flush call rows + meta. Call this **before** - harvesting so the temp dir is not deleted mid-finalization. - - Returns True if meta was observed. - """ - # Local import keeps drivers import-light for non-proxy unit tests. - from evals.proxy import SHUTDOWN_DEADLINE_S - - if max_wait_s is None: - max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 - deadline = time.monotonic() + max_wait_s - while True: - _, status = load_proxy_sidecar(sidecar_path) - if status.get("meta") is not None: - return True - rem = deadline - time.monotonic() - if rem <= 0: - break - time.sleep(min(poll_s, rem)) - _, status = load_proxy_sidecar(sidecar_path) - return status.get("meta") is not None - - -def harvest_proxy_after_cli_timeout( - calls: list[dict[str, Any]], - client_calls: list[dict[str, Any]], - sidecar_path: Path, - notes: list[str], - *, - max_wait_s: float | None = None, -) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: - """Wait for proxy finalization after CLI kill, then harvest the sidecar. - - If meta never appears within the wait window, harvest anyway (incomplete - note from ``apply_proxy_sidecar``). ``max_wait_s`` defaults to - ``SHUTDOWN_DEADLINE_S + 2`` (see ``wait_for_proxy_meta``). - """ - found = wait_for_proxy_meta(sidecar_path, max_wait_s=max_wait_s) - if not found: - notes.append("proxy_meta_wait_timeout") - return apply_proxy_sidecar(calls, client_calls, sidecar_path, notes) - - -# Bounded drain after process-group kill so communicate() never hangs forever -# when a grandchild still holds the pipe open. -_CLI_TIMEOUT_DRAIN_S = 2.0 - - -def _kill_process_group(proc: subprocess.Popen[Any]) -> bool: - """SIGKILL the process group whose leader is ``proc``. - - With ``start_new_session=True``, ``pgid == proc.pid`` even after the leader - has been reaped — call ``killpg(proc.pid, …)`` directly (never fall back to - killing only the leader, which leaves grandchildren alive). - - Returns True if ``killpg`` delivered the signal; False if the group is - already fully gone (``ProcessLookupError`` = success for cleanup, but the - kill itself did not run). - """ - if proc.pid is None: - return False - try: - # Do NOT use getpgid: if the leader is already reaped, getpgid fails and - # a proc.kill() fallback would recreate the original orphan bug. - os.killpg(proc.pid, signal.SIGKILL) - return True - except ProcessLookupError: - # No process left in the group — fully gone. - return False - - -def _decode_pipe(data: str | bytes | None, *, text: bool) -> str | bytes | None: - if data is None or not text or isinstance(data, str): - return data - return data.decode("utf-8", errors="replace") - - -def _close_pipes_and_reap(proc: subprocess.Popen[Any], *, drain_s: float = _CLI_TIMEOUT_DRAIN_S) -> tuple[Any, Any]: - """Bounded drain / close after a group kill. Never hangs unbounded.""" - try: - return proc.communicate(timeout=drain_s) - except (subprocess.TimeoutExpired, ValueError, OSError): - for stream in (proc.stdout, proc.stderr): - if stream is not None: - try: - stream.close() - except Exception: - pass - try: - proc.wait(timeout=1.0) - except subprocess.TimeoutExpired: - pass - return None, None - - -def run_cli_subprocess( - cmd: list[str], - *, - timeout: float | None = None, - cwd: str | None = None, - env: dict[str, str] | None = None, - capture_output: bool = True, - text: bool = True, - **_kwargs: Any, -) -> subprocess.CompletedProcess[Any]: - """Run a CLI in its own process group; kill the **whole group** on timeout/interrupt. - - Node wrappers (e.g. ``codex``) spawn native grandchildren. Plain - ``subprocess.run`` on timeout only kills the parent; the grandchild keeps - stdout open and ``communicate()`` hangs indefinitely. This runner: - - 1. launches with ``start_new_session=True`` (new process group; pgid=pid); - 2. on timeout **or any other exception** (incl. KeyboardInterrupt), - ``os.killpg(pid, SIGKILL)`` the group; - 3. drains pipes with a **bounded** second ``communicate`` (never unbounded). - - Raises ``subprocess.TimeoutExpired`` with attribute - ``killed_process_group=True`` only when killpg actually delivered the signal. - """ - popen_kwargs: dict[str, Any] = { - "cwd": cwd, - "start_new_session": True, - "stdout": subprocess.PIPE if capture_output else None, - "stderr": subprocess.PIPE if capture_output else None, - "text": text, - } - if env is not None: - popen_kwargs["env"] = env - - proc = subprocess.Popen(cmd, **popen_kwargs) # noqa: S603 — eval harness launches user CLIs - try: - stdout, stderr = proc.communicate(timeout=timeout) - except subprocess.TimeoutExpired as exc: - killed = _kill_process_group(proc) - out, err = _close_pipes_and_reap(proc) - if out is None and err is None: - stdout = _decode_pipe(exc.stdout, text=text) or ("" if text else b"") - stderr = _decode_pipe(exc.stderr, text=text) or ("" if text else b"") - else: - stdout, stderr = out, err - te = subprocess.TimeoutExpired( - cmd=cmd, - timeout=timeout if timeout is not None else 0, - output=stdout, - stderr=stderr, - ) - te.killed_process_group = killed # type: ignore[attr-defined] - raise te from None - except BaseException: - # KeyboardInterrupt / SystemExit / etc. — do not leave the CLI tree running. - # start_new_session means SIGINT no longer reaches the group automatically. - _kill_process_group(proc) - _close_pipes_and_reap(proc) - raise - - return subprocess.CompletedProcess(cmd, proc.returncode if proc.returncode is not None else 0, stdout, stderr) - - -def _note_timeout_kill(notes: list[str], exc: BaseException) -> None: - """Append process-group kill note when killpg actually delivered the signal.""" - if getattr(exc, "killed_process_group", False): - notes.append("timeout_killed_process_group") - - -@dataclass -class AgentRun: - """Normalized result of one agent task execution.""" - - # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} - calls: list[dict[str, Any]] - final_text: str - usage: dict[str, Any] | None - stopped_reason: str - raw_ref: str | None = None - # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics - client_tool_calls: list[dict[str, Any]] = field(default_factory=list) - # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens - usage_total: dict[str, Any] | None = None - # Harness extras (optional; defaults keep SDK path simple) - usage_scope: str = "run" # 'run' | 'iteration' - call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'sdk' - hit_max_turns: bool = False - wall_time_s: float = 0.0 - experimental: bool = False - notes: list[str] = field(default_factory=list) - - -class AgentDriver(Protocol): - """Pluggable agent backend for evals.run.""" - - name: str - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: ... - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def strip_mcp_prefix(name: str) -> str: - """Strip Claude/Codex MCP tool name prefixes for classification. - - Examples: - mcp__plane__list_work_items → list_work_items - mcp__plane-mcp-server__find_work_items → find_work_items - """ - if not name: - return name - m = _MCP_PREFIX_RE2.match(name) - if m: - return m.group(1) - return name - - -def is_plane_mcp_tool(name: str) -> bool: - """True when the raw tool name is from our Plane MCP server (pre-strip). - - Claude surfaces MCP tools as ``mcp____``. Our config registers - the server as ``plane``, so names look like ``mcp__plane__find_work_items``. - Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. - """ - if not name: - return False - # mcp__plane__tool or mcp__plane-foo__tool - return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") - - -def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: - """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" - raw = str(name or "") - if not isinstance(args, dict): - args = {"_raw": args} - if is_plane_mcp_tool(raw): - return { - "tool": strip_mcp_prefix(raw), - "args": args, - "origin": "plane", - "raw_tool": raw, - } - return { - "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) - "args": args, - "origin": "client", - "raw_tool": raw, - } - - -def split_plane_and_client_calls( - calls: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Partition tagged calls into plane vs client lists. - - Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls - (SDK path) default to plane so existing harness behavior is unchanged. - """ - plane: list[dict[str, Any]] = [] - client: list[dict[str, Any]] = [] - for c in calls: - origin = c.get("origin") - if origin is None: - raw = str(c.get("raw_tool") or c.get("tool") or "") - if is_plane_mcp_tool(raw): - origin = "plane" - elif raw.startswith("mcp__"): - origin = "client" # other MCP server - else: - origin = "plane" # bare name → assume plane (SDK) - if origin == "client": - client.append(c) - else: - plane.append(c) - return plane, client - - -def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: - """Parse Claude print-mode usage into (raw_usage, usage_total). - - Real envelope (probed 2026-08-12, ``claude -p --output-format json``):: - - { - "usage": { - "input_tokens": 10, # uncached NEW input only — NOT run total - "cache_creation_input_tokens": 17459, - "cache_read_input_tokens": 18464, - "output_tokens": 143, - "iterations": [...], - ... - }, - "modelUsage": { - "": { - "inputTokens": 10, - "outputTokens": 143, - "cacheReadInputTokens": 18464, - "cacheCreationInputTokens": 17459, - "costUSD": 0.037, - ... - } - }, - "total_cost_usd": 0.037 - } - - ``usage.input_tokens`` alone is misleading for multi-turn cached runs (live - rows showed 8–10 while cache_read was 180k+). We keep the split fields and - compute an inclusive total under ``usage_total``; callers must **not** copy - bare ``input_tokens`` into ``cum_input_tokens``. - """ - usage = data.get("usage") - if usage is not None and not isinstance(usage, dict): - usage = None - model_usage = data.get("modelUsage") or data.get("model_usage") - if model_usage is not None and not isinstance(model_usage, dict): - model_usage = None - cost = data.get("total_cost_usd") - if cost is None and isinstance(usage, dict): - cost = usage.get("total_cost_usd") - - if usage is None and model_usage is None and cost is None: - return None, None - - raw = dict(usage or {}) - if cost is not None: - raw["total_cost_usd"] = cost - if model_usage is not None: - raw["modelUsage"] = model_usage - - # Prefer summing modelUsage (per-model run totals) when present - sum_in = sum_out = sum_cr = sum_cc = sum_cost = 0.0 - used_model_usage = False - if model_usage: - for _mid, mu in model_usage.items(): - if not isinstance(mu, dict): - continue - used_model_usage = True - sum_in += float(mu.get("inputTokens") or mu.get("input_tokens") or 0) - sum_out += float(mu.get("outputTokens") or mu.get("output_tokens") or 0) - sum_cr += float(mu.get("cacheReadInputTokens") or mu.get("cache_read_input_tokens") or 0) - sum_cc += float(mu.get("cacheCreationInputTokens") or mu.get("cache_creation_input_tokens") or 0) - sum_cost += float(mu.get("costUSD") or mu.get("cost_usd") or 0) - - if used_model_usage: - uncached_in = int(sum_in) - out_tok = int(sum_out) - cache_read = int(sum_cr) - cache_write = int(sum_cc) - total_cost = float(sum_cost) if sum_cost else cost - else: - uncached_in = int(raw.get("input_tokens") or 0) - out_tok = int(raw.get("output_tokens") or 0) - cache_read = int(raw.get("cache_read_input_tokens") or 0) - cache_write = int(raw.get("cache_creation_input_tokens") or 0) - total_cost = cost - - usage_total: dict[str, Any] = { - "input_tokens": uncached_in, # uncached / new tokens only - "output_tokens": out_tok, - "cache_read_input_tokens": cache_read, - "cache_creation_input_tokens": cache_write, - "total_input_tokens_including_cache": uncached_in + cache_read + cache_write, - "total_cost_usd": total_cost, - "modelUsage": model_usage, - "source": "modelUsage" if used_model_usage else "usage", - } - return raw, usage_total - - -def _claude_project_dir(cwd: Path) -> Path: - """Map a cwd to ``~/.claude/projects/`` (``/`` → ``-``).""" - munged = str(cwd.resolve()).replace("/", "-") - return Path.home() / ".claude" / "projects" / munged - - -def parse_claude_json_result(payload: dict[str, Any] | str) -> dict[str, Any]: - """Extract final text, usage, session id, num_turns from ``claude -p --output-format json``. - - The print-mode JSON envelope is a single object (``type=result``) with - ``result``, ``session_id``, ``num_turns``, ``total_cost_usd``, ``usage``, - and ``modelUsage``. Per-call tool detail is usually **absent** — callers - should fall back to the session transcript. - """ - if isinstance(payload, str): - payload = json.loads(payload) - if not isinstance(payload, dict): - raise ValueError(f"expected JSON object from claude, got {type(payload)}") - - data = payload - - final = data.get("result") - if final is None: - final = data.get("final_text") or data.get("text") or "" - if not isinstance(final, str): - final = json.dumps(final, default=str) - - usage, usage_total = normalize_claude_usage(data) - - session_id = data.get("session_id") or data.get("sessionId") - num_turns = data.get("num_turns") - if num_turns is None: - num_turns = data.get("numTurns") - is_error = bool(data.get("is_error") or data.get("isError")) - subtype = data.get("subtype") or "" - stop_reason = data.get("stop_reason") or data.get("terminal_reason") or "" - - # Tool calls rarely present in the result envelope; collect if present. - calls: list[dict[str, Any]] = [] - for key in ("tool_calls", "tools", "calls"): - raw = data.get(key) - if isinstance(raw, list): - for item in raw: - if not isinstance(item, dict): - continue - name = item.get("name") or item.get("tool") or "" - args = item.get("input") or item.get("arguments") or item.get("args") or {} - if isinstance(args, str): - try: - args = json.loads(args) - except json.JSONDecodeError: - args = {"_raw": args} - calls.append(normalize_tool_call(str(name), args)) - - # Preserve Claude error subtypes (e.g. error_during_execution, error_max_turns). - # is_error alone collapses to "error" and loses the subtype run.py uses for infra_cli. - if is_error and subtype and str(subtype) not in ("success", ""): - stopped = str(subtype) - elif is_error: - stopped = "error" - else: - stopped = str(stop_reason) if stop_reason else "end_turn" - if subtype and subtype not in ("success", "") and stopped == "end_turn": - stopped = str(subtype) - - plane_calls, client_calls = split_plane_and_client_calls(calls) - - return { - "final_text": final, - "usage": usage, - "usage_total": usage_total, - "session_id": session_id, - "num_turns": int(num_turns) if num_turns is not None else None, - "calls": plane_calls, - "client_tool_calls": client_calls, - "stopped_reason": stopped, - "raw": data, - } - - -def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]]: - """Parse ``tool_use`` blocks from a Claude Code session JSONL transcript. - - Returns tagged calls (``origin`` plane|client). Use - ``split_plane_and_client_calls`` before classification. - """ - calls: list[dict[str, Any]] = [] - if not transcript_path.is_file(): - return calls - with transcript_path.open(encoding="utf-8") as fh: - for line in fh: - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - msg = row.get("message") if isinstance(row, dict) else None - if not isinstance(msg, dict): - if row.get("type") == "assistant" and isinstance(row.get("content"), list): - content = row["content"] - else: - continue - else: - content = msg.get("content") - if not isinstance(content, list): - continue - for block in content: - if not isinstance(block, dict): - continue - if block.get("type") != "tool_use": - continue - name = str(block.get("name") or "") - args = block.get("input") or {} - if not isinstance(args, dict): - args = {"_raw": args} - calls.append(normalize_tool_call(name, args)) - return calls - - -def find_claude_transcript(session_id: str | None, cwd: Path) -> Path | None: - """Locate ``~/.claude/projects//.jsonl``.""" - if not session_id: - return None - candidate = _claude_project_dir(cwd) / f"{session_id}.jsonl" - if candidate.is_file(): - return candidate - # Fallback: scan project dir for a file containing the session id - proj = _claude_project_dir(cwd) - if not proj.is_dir(): - return None - direct = proj / f"{session_id}.jsonl" - if direct.is_file(): - return direct - for p in proj.glob("*.jsonl"): - if session_id in p.name: - return p - return None - - -def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: - if isinstance(raw_args, str): - try: - args = json.loads(raw_args) - except json.JSONDecodeError: - return {"_raw": raw_args} - return args if isinstance(args, dict) else {"_raw": args} - if isinstance(raw_args, dict): - return raw_args - return {"_raw": raw_args} - - -def parse_codex_jsonl_events(lines: list[str] | str) -> dict[str, Any]: - """Parse ``codex exec --json`` stdout (JSONL) for function_call + final text + usage. - - Supports both schemas: - - **v0.147+ streamable**: ``thread.started`` / ``item.completed`` / ``turn.completed`` - (``thread_id`` matches rollout filename suffix). - - **Legacy**: ``session_meta`` / ``response_item`` / ``event_msg`` payloads. - New keys are tried first; legacy handling is retained. - """ - if isinstance(lines, str): - lines = lines.splitlines() - calls: list[dict[str, Any]] = [] - final_parts: list[str] = [] - usage: dict[str, Any] | None = None - stopped = "end_turn" - session_id: str | None = None - - for line in lines: - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - rtype = row.get("type") - - # --- New schema (codex exec --json v0.147+): try first --- - if rtype == "thread.started": - session_id = row.get("thread_id") or session_id - continue - if rtype == "turn.started": - continue - if rtype == "item.completed": - item = row.get("item") if isinstance(row.get("item"), dict) else {} - itype = item.get("type") - if itype == "agent_message": - text = item.get("text") - if text: - final_parts.append(str(text)) - elif itype in ( - "function_call", - "tool_call", - "mcp_tool_call", - "command_execution", - "file_change", - ): - # Proxy sidecar is primary for plane calls; harvest best-effort names. - name = str(item.get("name") or item.get("tool") or item.get("command") or itype) - args = item.get("arguments") or item.get("args") or item.get("input") or {} - calls.append(normalize_tool_call(name, _codex_parse_tool_args(args))) - continue - if rtype == "turn.completed": - u = row.get("usage") if isinstance(row.get("usage"), dict) else {} - if u: - usage = { - "input_tokens": u.get("input_tokens", 0) or 0, - "output_tokens": u.get("output_tokens", 0) or 0, - "cache_read_input_tokens": u.get("cached_input_tokens", 0) or 0, - "cache_creation_input_tokens": u.get("cache_write_input_tokens", 0) or 0, - "total_tokens": u.get("total_tokens"), - } - continue - - # --- Legacy schema --- - payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} - - if rtype == "session_meta": - session_id = payload.get("id") or session_id - continue - - if rtype == "response_item": - pt = payload.get("type") - if pt == "function_call": - name = str(payload.get("name") or "") - calls.append(normalize_tool_call(name, _codex_parse_tool_args(payload.get("arguments") or "{}"))) - elif pt == "message": - # Assistant final-ish content - role = payload.get("role") - content = payload.get("content") - if role == "assistant" and isinstance(content, list): - for c in content: - if isinstance(c, dict) and c.get("type") in ("output_text", "text"): - t = c.get("text") or c.get("output_text") - if t: - final_parts.append(str(t)) - continue - - if rtype == "event_msg": - pt = payload.get("type") - if pt == "agent_message": - msg = payload.get("message") or payload.get("text") - if msg: - final_parts.append(str(msg)) - elif pt == "token_count": - info = payload.get("info") or {} - total = info.get("total_token_usage") or info.get("last_token_usage") or {} - if isinstance(total, dict): - usage = { - "input_tokens": total.get("input_tokens", 0) or 0, - "output_tokens": total.get("output_tokens", 0) or 0, - "cache_read_input_tokens": total.get("cached_input_tokens", 0) or 0, - "cache_creation_input_tokens": total.get("cache_write_input_tokens", 0) or 0, - "total_tokens": total.get("total_tokens"), - } - elif pt == "task_complete": - stopped = "end_turn" - elif pt == "turn_aborted": - stopped = "aborted" - continue - - plane_calls, client_calls = split_plane_and_client_calls(calls) - return { - "calls": plane_calls, - "client_tool_calls": client_calls, - "final_text": "\n".join(final_parts).strip(), - "usage": usage, - "stopped_reason": stopped, - "session_id": session_id, - } - - -def parse_codex_rollout_calls(rollout_path: Path) -> list[dict[str, Any]]: - """Parse function_call records from a Codex session rollout JSONL (plane only).""" - if not rollout_path.is_file(): - return [] - lines = rollout_path.read_text(encoding="utf-8").splitlines() - return parse_codex_jsonl_events(lines)["calls"] - - -def find_codex_rollout(session_id: str | None, *, after_ts: float | None = None) -> Path | None: - """Find a rollout JSONL under ``~/.codex/sessions`` matching *session_id* exactly. - - Matches filename containing the id (rollout filenames end with ``thread_id``) - or a first-line ``session_meta`` / ``thread.started`` id field. - - **No newest-after-ts fallback**: under parallel runs that would pick another - task's rollout and corrupt final_text. Callers should note - ``codex_rollout_unmatched`` when this returns None. - - ``after_ts`` is accepted for API compatibility but ignored. - """ - del after_ts # intentionally unused — see docstring - if not session_id: - return None - root = Path.home() / ".codex" / "sessions" - if not root.is_dir(): - return None - sid = str(session_id) - for p in root.rglob("*.jsonl"): - if sid in p.name: - return p - for p in root.rglob("rollout-*.jsonl"): - try: - with p.open(encoding="utf-8") as fh: - first = fh.readline() - row = json.loads(first) - except Exception: - continue - # Legacy session_meta - payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} - if payload.get("id") == sid: - return p - # New thread.started on first line (unusual but possible) - if row.get("type") == "thread.started" and row.get("thread_id") == sid: - return p - if row.get("thread_id") == sid or row.get("session_id") == sid: - return p - return None - - -def write_claude_mcp_config( - path: Path, - *, - command: str, - args: list[str], - env: dict[str, str], - server_name: str = "plane", -) -> None: - """Write a Claude-compatible mcp-config JSON file.""" - cfg = { - "mcpServers": { - server_name: { - "command": command, - "args": args, - "env": env, - } - } - } - path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") - - -def write_codex_mcp_override_args( - *, - command: str, - args: list[str], - env: dict[str, str], - server_name: str = "plane", -) -> list[str]: - """Build ``codex exec -c ...`` overrides for a stdio MCP server. - - Codex stores MCP under ``[mcp_servers.]`` with ``command``, ``args``, - and ``env`` (see user config.toml). Overrides use dotted ``-c`` paths. - """ - out: list[str] = [ - "-c", - f"mcp_servers.{server_name}.command={json.dumps(command)}", - "-c", - f"mcp_servers.{server_name}.args={json.dumps(args)}", - ] - # env table — pass each key - for k, v in env.items(): - out.extend(["-c", f"mcp_servers.{server_name}.env.{k}={json.dumps(v)}"]) - return out - - -# --------------------------------------------------------------------------- -# Claude CLI driver -# --------------------------------------------------------------------------- - - -class ClaudeCliDriver: - """Run tasks via ``claude -p`` on the user's Claude Code subscription.""" - - name = "claude-cli" - - def __init__( - self, - *, - claude_bin: str = "claude", - python_bin: str | None = None, - permission_mode: str = "bypassPermissions", - strict_mcp: bool = True, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - server_command: list[str] | None = None, - use_proxy: bool = True, - ) -> None: - self.claude_bin = claude_bin - self.python_bin = python_bin or sys.executable - self.permission_mode = permission_mode - self.strict_mcp = strict_mcp - self._runner = runner or run_cli_subprocess - # Full replacement for the MCP server launch (external surfaces under - # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = [] - t0 = time.perf_counter() - - with tempfile.TemporaryDirectory(prefix="plane-eval-claude-") as td: - td_path = Path(td) - mcp_cfg = td_path / "mcp.json" - sidecar = td_path / "proxy-sidecar.jsonl" - # Only pass Plane-related env into the MCP child (plus PATH/HOME if present). - child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env = ensure_proxy_pythonpath(child_env) - else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - write_claude_mcp_config( - mcp_cfg, - command=server_cmd, - args=server_args, - env=child_env, - server_name="plane", - ) - - cmd: list[str] = [ - self.claude_bin, - "-p", - "--output-format", - "json", - "--mcp-config", - str(mcp_cfg), - "--permission-mode", - self.permission_mode, - "--max-turns", - str(max_turns), - ] - if self.strict_mcp: - cmd.append("--strict-mcp-config") - if model: - cmd.extend(["--model", model]) - if system: - cmd.extend(["--append-system-prompt", system]) - # Allow MCP tools from our server without interactive prompts - # --allowedTools is variadic and would swallow the trailing prompt; use = form. - cmd.append("--allowedTools=mcp__plane__*") - cmd.append(prompt) - - timeout_s = max(120, max_turns * 60) - try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - _note_timeout_kill(notes, exc) - # Wait for proxy finalization before harvesting / temp dir teardown. - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = harvest_proxy_after_cli_timeout( - calls, client_calls, sidecar, notes - ) - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - - parsed: dict[str, Any] | None = None - parse_err: str | None = None - # JSON may be the whole stdout or the last JSON object line - for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): - if not candidate or not candidate.lstrip().startswith("{"): - continue - try: - parsed = parse_claude_json_result(candidate) - break - except (json.JSONDecodeError, ValueError, TypeError) as exc: - parse_err = str(exc) - continue - - if parsed is None: - notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") - if proc.returncode != 0: - notes.append(f"claude_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - if self.use_proxy: - apply_proxy_sidecar([], [], sidecar, notes) - detail = "; ".join(notes) - raise RuntimeError(f"claude cli failed: {detail}") - - # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). - if proc.returncode != 0: - notes.append(f"claude_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - - # JSON rarely embeds per-call tool detail — prefer transcript when present. - calls = list(parsed.get("calls") or []) - client_calls = list(parsed.get("client_tool_calls") or []) - call_source = "json" if (calls or client_calls) else "json" - session_id = parsed.get("session_id") - transcript = find_claude_transcript(session_id, cwd) - if transcript is not None: - tagged = parse_claude_transcript_calls(transcript) - t_plane, t_client = split_plane_and_client_calls(tagged) - if t_plane or t_client: - calls, client_calls = t_plane, t_client - call_source = "transcript" - notes.append(f"calls_from_transcript:{transcript}") - if not calls and not client_calls: - notes.append("no_tool_calls_in_json_or_transcript") - - # Proxy sidecar (when enabled) replaces CLI-parsed plane calls. - if self.use_proxy: - calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proxy_src == "proxy": - call_source = "proxy" - - num_turns = parsed.get("num_turns") - hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) - stopped = parsed["stopped_reason"] - if hit_max and stopped in ("end_turn", "completed", ""): - stopped = "max_turns" - - raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=parsed["final_text"], - usage=parsed.get("usage"), - usage_total=parsed.get("usage_total"), - stopped_reason=stopped, - raw_ref=raw_ref, - usage_scope="run", - call_source=call_source, - hit_max_turns=hit_max, - wall_time_s=round(wall, 3), - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# Codex CLI driver (experimental — do not spend live quota from CI) -# --------------------------------------------------------------------------- - - -class CodexCliDriver: - """Run tasks via ``codex exec`` (experimental; metered quota). - - Live invocation is supported for the interface, but the eval harness should - only exercise this driver when the team explicitly opts in. Offline tests - inject a fake runner and never touch the real binary. - """ - - name = "codex-cli" - experimental = True - - def __init__( - self, - *, - codex_bin: str = "codex", - python_bin: str | None = None, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - allow_live: bool = False, - server_command: list[str] | None = None, - use_proxy: bool = True, - ) -> None: - self.codex_bin = codex_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.allow_live = allow_live - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes = ["experimental:codex-cli"] - if self._runner is run_cli_subprocess and not self.allow_live: - raise RuntimeError( - "CodexCliDriver refuses live runs by default (metered weekly quota). " - "Pass allow_live=True or inject a fake runner for tests." - ) - - child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - with tempfile.TemporaryDirectory(prefix="plane-eval-codex-") as td: - td_path = Path(td) - sidecar = td_path / "proxy-sidecar.jsonl" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env = ensure_proxy_pythonpath(child_env) - else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - mcp_args = write_codex_mcp_override_args( - command=server_cmd, - args=server_args, - env=child_env, - server_name="plane", - ) - cmd: list[str] = [ - self.codex_bin, - "exec", - "--json", - "--skip-git-repo-check", - *mcp_args, - ] - if model: - cmd.extend(["-m", model]) - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd.append(full_prompt) - - t0 = time.perf_counter() - timeout_s = max(120, max_turns * 60) - try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - _note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "stream" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - experimental=True, - notes=notes, - ) - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - parsed = parse_codex_jsonl_events(stdout) - calls = list(parsed.get("calls") or []) - client_calls = list(parsed.get("client_tool_calls") or []) - call_source = "stream" - session_id = parsed.get("session_id") - - # Exact-match rollout only — never steal another parallel task's file. - need_rollout = (not calls and not client_calls) or not parsed.get("final_text") - if need_rollout and session_id: - rollout = find_codex_rollout(session_id) - if rollout is not None: - full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) - if not calls and not client_calls: - calls = list(full.get("calls") or []) - client_calls = list(full.get("client_tool_calls") or []) - if calls or client_calls: - call_source = "transcript" - notes.append(f"calls_from_rollout:{rollout}") - if full.get("final_text") and not parsed.get("final_text"): - parsed["final_text"] = full["final_text"] - notes.append(f"final_text_from_rollout:{rollout}") - if full.get("usage") and not parsed.get("usage"): - parsed["usage"] = full["usage"] - else: - notes.append("codex_rollout_unmatched") - elif need_rollout and not session_id: - notes.append("codex_rollout_unmatched") - - if self.use_proxy: - calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proxy_src == "proxy": - call_source = "proxy" - - if proc.returncode != 0: - notes.append(f"codex_exit={proc.returncode}") - - usage = parsed.get("usage") - usage_total = None - if isinstance(usage, dict): - usage_total = { - "input_tokens": usage.get("input_tokens"), - "output_tokens": usage.get("output_tokens"), - "cache_read_input_tokens": usage.get("cache_read_input_tokens"), - "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), - "total_input_tokens_including_cache": ( - int(usage.get("input_tokens") or 0) - + int(usage.get("cache_read_input_tokens") or 0) - + int(usage.get("cache_creation_input_tokens") or 0) - ), - "source": "codex_token_count", - } - - raw_ref = f"session:{session_id}" if session_id else None - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=parsed.get("final_text") or "", - usage=usage, - usage_total=usage_total, - stopped_reason=parsed.get("stopped_reason") or "end_turn", - raw_ref=raw_ref, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, # codex exec has no max-turns flag in --help - wall_time_s=round(wall, 3), - experimental=True, - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# Antigravity CLI (agy) — proxy-first -# --------------------------------------------------------------------------- - - -def write_antigravity_mcp_config( - path: Path, - *, - command: str, - args: list[str], - env: dict[str, str], - server_name: str = "plane", -) -> None: - """Write ``mcpServers`` map JSON (Antigravity / agy mcp_config shape).""" - cfg = { - "mcpServers": { - server_name: { - "command": command, - "args": args, - "env": env, - } - } - } - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") - - -def prepare_antigravity_fake_home( - fake_home: Path, - *, - command: str, - args: list[str], - env: dict[str, str], - real_home: Path | None = None, -) -> None: - """Build an isolated HOME for agy with MCP config + shared auth artifacts. - - Writes mcp_config.json to BOTH documented locations (cheap; live probe - should settle which path agy actually reads): - - ~/.gemini/config/mcp_config.json - - ~/.gemini/antigravity-cli/mcp_config.json - - Creates ``antigravity-cli`` as a **real directory** (never a symlink of the - whole tree — that would write mcp_config and runtime logs into real user - state). Auth artifacts (``antigravity-oauth-token``) are plain **copies** — - never symlinks — so an in-place token refresh cannot write through into the - real home. Staleness over a single eval run is negligible. - """ - real_home = real_home or Path.home() - gemini_root = fake_home / ".gemini" - gemini_root.mkdir(parents=True, exist_ok=True) - - real_cli = real_home / ".gemini" / "antigravity-cli" - fake_cli = gemini_root / "antigravity-cli" - # Always a real directory — never symlink the whole tree. - if fake_cli.is_symlink() or fake_cli.is_file(): - fake_cli.unlink() - fake_cli.mkdir(parents=True, exist_ok=True) - - # Share auth via plain COPY only — never symlink (in-place token refresh - # must not write through into the real home). - if real_cli.is_dir(): - for name in ("antigravity-oauth-token",): - src = real_cli / name - dst = fake_cli / name - if src.is_file() and not dst.exists(): - try: - dst.write_bytes(src.read_bytes()) - except OSError: - pass - - # Dual write as real files (not through any symlink). - for rel in ( - Path(".gemini") / "config" / "mcp_config.json", - Path(".gemini") / "antigravity-cli" / "mcp_config.json", - ): - write_antigravity_mcp_config( - fake_home / rel, - command=command, - args=args, - env=env, - server_name="plane", - ) - - -class AntigravityCliDriver: - """Run tasks via Google Antigravity CLI (``agy``). - - Probed flags (2026-08-12, ``agy --help``): - - ``-p`` / ``--print`` headless single-prompt mode - - ``--output-format`` text|json|stream-json - - ``--model``, ``--dangerously-skip-permissions`` - - MCP via ``~/.gemini/config/mcp_config.json`` (``mcpServers`` map; - stdio: command/args/env). No CLI flag for MCP config → HOME isolation. - - No max-turns / turn-cap flag in help → ``hit_max_turns=False`` + note. - - Tool calls come from the recording proxy sidecar (protocol-layer), not - agy stdout parsing. - """ - - name = "antigravity-cli" - - def __init__( - self, - *, - agy_bin: str = "agy", - python_bin: str | None = None, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - server_command: list[str] | None = None, - use_proxy: bool = True, - ) -> None: - self.agy_bin = agy_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = ["no_turn_cap"] - t0 = time.perf_counter() - child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - - with tempfile.TemporaryDirectory(prefix="plane-eval-antigravity-") as td: - td_path = Path(td) - sidecar = td_path / "proxy-sidecar.jsonl" - fake_home = td_path / "home" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env_plane = ensure_proxy_pythonpath(child_env_plane) - else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - prepare_antigravity_fake_home( - fake_home, - command=server_cmd, - args=server_args, - env=child_env_plane, - ) - - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd: list[str] = [ - self.agy_bin, - "-p", - "--output-format", - "json", - "--dangerously-skip-permissions", - ] - if model: - cmd.extend(["--model", model]) - cmd.append(full_prompt) - - run_env = {**os.environ, "HOME": str(fake_home)} - if "PATH" in child_env_plane: - run_env["PATH"] = child_env_plane["PATH"] - - timeout_s = max(120, max_turns * 60) - try: - try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - env=run_env, - ) - except TypeError: - # Some test runners reject ``env=``; retry without it. - # TimeoutExpired from this path must still hit the harvest below. - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - _note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - usage_scope="run", - call_source=call_source, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proc.returncode != 0: - notes.append(f"agy_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - - final_text = stdout.strip() - try: - if final_text.lstrip().startswith("{"): - blob = json.loads(final_text) - if isinstance(blob, dict): - final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) - except json.JSONDecodeError: - pass - - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=final_text, - usage=None, - stopped_reason="error" if proc.returncode else "end_turn", - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# OpenCode CLI — proxy-first -# --------------------------------------------------------------------------- - - -def write_opencode_mcp_config( - path: Path, - *, - command: list[str], - env: dict[str, str], - server_name: str = "plane", -) -> None: - """Write project ``opencode.json`` with a local MCP server entry. - - Schema (opencode.ai docs / probed binary strings, 2026-08-12):: - - {"mcp": {"plane": {"type": "local", "command": [...], "environment": {...}}}} - """ - cfg = { - "$schema": "https://opencode.ai/config.json", - "mcp": { - server_name: { - "type": "local", - "command": list(command), - "environment": env, - "enabled": True, - } - }, - } - path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") - - -class OpencodeCliDriver: - """Run tasks via ``opencode run`` (proxy-first call recording). - - Probed flags (2026-08-12): - - ``opencode run [message..]`` non-interactive - - ``--format json|default``, ``-m/--model`` - - MCP via ``opencode.json`` ``mcp`` section (local: type/command/environment) - written into the task cwd (or a temp project dir). - - No turn-cap flag → ``hit_max_turns=False`` + note ``no_turn_cap``. - """ - - name = "opencode-cli" - - def __init__( - self, - *, - opencode_bin: str = "opencode", - python_bin: str | None = None, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - server_command: list[str] | None = None, - use_proxy: bool = True, - ) -> None: - self.opencode_bin = opencode_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - base_cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = ["no_turn_cap"] - t0 = time.perf_counter() - child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - - with tempfile.TemporaryDirectory(prefix="plane-eval-opencode-", dir=str(base_cwd)) as td: - # Project-local opencode.json so we do not pollute the user's global config. - proj = Path(td) - sidecar = proj / "proxy-sidecar.jsonl" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - launch = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) - child_env_plane = ensure_proxy_pythonpath(child_env_plane) - else: - launch = real_cmd - write_opencode_mcp_config( - proj / "opencode.json", - command=launch, - env=child_env_plane, - server_name="plane", - ) - - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd: list[str] = [ - self.opencode_bin, - "run", - "--format", - "json", - ] - if model: - cmd.extend(["-m", model]) - cmd.append(full_prompt) - - timeout_s = max(120, max_turns * 60) - try: - proc = self._runner( - cmd, - cwd=str(proj), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - _note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - usage_scope="run", - call_source=call_source, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proc.returncode != 0: - notes.append(f"opencode_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - - final_text = stdout.strip() - # JSONL events: concatenate text-ish fields best-effort. - if final_text and "\n" in final_text: - parts: list[str] = [] - for line in final_text.splitlines(): - line = line.strip() - if not line.startswith("{"): - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(row, dict): - for key in ("text", "message", "part", "delta"): - v = row.get(key) - if isinstance(v, str) and v.strip(): - parts.append(v) - if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): - parts.append(row["content"]) - if parts: - final_text = "\n".join(parts) - - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=final_text, - usage=None, - stopped_reason="error" if proc.returncode else "end_turn", - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# Registry -# --------------------------------------------------------------------------- - -KNOWN_DRIVERS = frozenset({"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) - - -def get_driver(name: str, **kwargs: Any) -> AgentDriver | None: - """Return a driver instance, or None for the in-process ``sdk`` path.""" - key = (name or "sdk").strip().lower() - if key == "sdk": - return None # handled inline in evals.run - if key == "claude-cli": - return ClaudeCliDriver(**kwargs) - if key == "codex-cli": - return CodexCliDriver(**kwargs) - if key == "antigravity-cli": - return AntigravityCliDriver(**kwargs) - if key == "opencode-cli": - return OpencodeCliDriver(**kwargs) - raise ValueError(f"unknown driver {name!r}; expected one of {sorted(KNOWN_DRIVERS)}") - - -def agent_run_to_harness_dict( - run: AgentRun, - *, - optimal: set[str], - alternate: set[str], - classify: Callable[[str, set[str], set[str]], str], - skip_result_tokens: bool = True, -) -> dict[str, Any]: - """Map an ``AgentRun`` onto the dict shape expected by ``run_live`` rows. - - Only **plane** MCP tools are classified and counted in ``num_calls`` / - mispick metrics. Client built-ins (``ToolSearch``, …) go to - ``client_tool_calls`` and are excluded. - - CLI drivers never populate ``cum_input_tokens`` from bare - ``usage.input_tokens`` (that field is uncached-only under Claude Code and - misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. - """ - # Re-split in case callers passed a mixed list - plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) - client_src = list(run.client_tool_calls) + client_extra - - calls: list[dict[str, Any]] = [] - for c in plane_src: - tool = c.get("tool") or "" - args = c.get("args") or {} - try: - args_chars = len(json.dumps(args, default=str)) - except Exception: - args_chars = len(str(args)) - rec: dict[str, Any] = { - "tool": tool, - "class": classify(str(tool), optimal, alternate), - "args_chars": args_chars, - "result_tokens": None, - "result_chars": int(c["result_chars"]) if c.get("result_chars") is not None else 0, - "result_kind": "text", - "is_error": bool(c.get("is_error")), - } - if c.get("duration_ms") is not None: - rec["duration_ms"] = c["duration_ms"] - # Action-dispatch surfaces: the action arg IS the second half of the - # tool choice — keep it (args content is otherwise not persisted). - if isinstance(args, dict) and isinstance(args.get("action"), str): - rec["action"] = args["action"] - if skip_result_tokens: - rec["result_tokens_skipped"] = "no API key / CLI driver has no count_tokens" - calls.append(rec) - - client_tool_calls: list[dict[str, Any]] = [] - for c in client_src: - tool = c.get("tool") or c.get("raw_tool") or "" - args = c.get("args") or {} - try: - args_chars = len(json.dumps(args, default=str)) - except Exception: - args_chars = len(str(args)) - client_tool_calls.append( - { - "tool": tool, - "args_chars": args_chars, - "raw_tool": c.get("raw_tool") or tool, - } - ) - - stop_reason = run.stopped_reason - hit_max = run.hit_max_turns - if hit_max: - stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" - - errored = sum(1 for c in calls if c.get("is_error")) - alternate_n = sum(1 for c in calls if c["class"] == "alternate") - out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") - - # CLI path: never write misleading cum_input_tokens from uncached-only field - is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" - usage_total = run.usage_total - if usage_total is None and isinstance(run.usage, dict) and is_cli: - # Best-effort rebuild if driver forgot usage_total - _, usage_total = normalize_claude_usage({"usage": run.usage, "modelUsage": run.usage.get("modelUsage")}) - - if is_cli and skip_result_tokens: - cum_input: int | None = None - cum_reason: str | None = ( - "CLI driver: Claude usage.input_tokens is uncached-only; " - "see usage_total (cache_read/cache_creation/output/cost) for run accounting" - ) - usage_per_iteration: list[dict[str, int]] = [] - else: - cum_input = 0 - cum_reason = None - usage_per_iteration = [] - if run.usage and run.usage_scope == "iteration": - pass # SDK fills this separately - - return { - "final_text": run.final_text, - "calls": calls, - "num_calls": len(calls), - "client_tool_calls": client_tool_calls, - "client_tool_call_count": len(client_tool_calls), - "errored_calls": errored, - "alternate_calls": alternate_n, - "out_of_set_calls": out_of_set_n, - "total_result_tokens": 0 - if skip_result_tokens - else sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None), - "usage_per_iteration": usage_per_iteration, - "cum_input_tokens": cum_input, - "cum_input_tokens_reason": cum_reason, - "wall_time_s": run.wall_time_s, - "stop_reason": stop_reason, - "hit_max_iterations": hit_max, - "result_pair_mismatch": False, - "token_count_failures": 0, - "usage_scope": run.usage_scope, - "call_source": run.call_source, - "driver_raw_ref": run.raw_ref, - "driver_notes": list(run.notes), - "result_tokens_skipped_reason": ( - "CLI driver: count_tokens requires Anthropic API key; skipped" if skip_result_tokens else None - ), - "usage": run.usage, - "usage_total": usage_total, - } diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py new file mode 100644 index 00000000..00c56d75 --- /dev/null +++ b/evals/drivers/__init__.py @@ -0,0 +1,142 @@ +"""Agent-driver abstraction for the Plane MCP eval harness. + +Drivers run one task against a tool surface and return a normalized +``AgentRun``. The default ``sdk`` driver preserves the historical Anthropic +SDK + in-process MCP client path. CLI drivers (``claude-cli``, ``codex-cli``, +``antigravity-cli``, ``opencode-cli``) spawn locally installed agent CLIs on +the user's subscription — no Anthropic API key required for those paths. + +Real CLI surfaces (probed on this machine, 2026-08-12): + +Claude Code (``claude`` v2.1.228): + - ``-p`` / ``--print`` headless + - ``--mcp-config `` (repeatable; ``--strict-mcp-config``) + - ``--output-format json|text|stream-json`` (print mode) + - ``--max-turns `` (print mode; *hidden* from ``--help`` but present) + - ``--model `` + - ``--permission-mode`` choices: acceptEdits, auto, bypassPermissions, + manual, dontAsk, plan + - ``--dangerously-skip-permissions``, ``--allowedTools`` / ``--allowed-tools`` + - Transcript: ``~/.claude/projects//.jsonl`` + with ``assistant`` rows whose ``message.content`` holds ``tool_use`` blocks. + - MCP tools surface as ``mcp____`` — strip for classification. + +Codex (``codex exec``): + - ``codex exec --json`` JSONL events on stdout + - ``-c key=value`` / ``--config`` for config.toml overrides (incl. mcp_servers) + - ``-m`` / ``--model`` + - Session rollouts: ``~/.codex/sessions/**/rollout-*.jsonl`` with + ``response_item`` / ``function_call`` payloads (name + arguments JSON string) + - Marked **experimental**; live runs are opt-in (metered quota). + +This package splits that surface into focused modules (base types, subprocess +lifecycle, recording-proxy glue, and per-vendor drivers). Import from +``evals.drivers`` as before — public names are re-exported here. +""" + +from __future__ import annotations + +from typing import Any + +from evals.drivers.antigravity import ( + AntigravityCliDriver, + prepare_antigravity_fake_home, + write_antigravity_mcp_config, +) +from evals.drivers.base import ( + REPO_ROOT, + AgentDriver, + AgentRun, + agent_run_to_harness_dict, + is_plane_mcp_tool, + normalize_tool_call, + split_plane_and_client_calls, + strip_mcp_prefix, +) +from evals.drivers.claude import ( + ClaudeCliDriver, + find_claude_transcript, + normalize_claude_usage, + parse_claude_json_result, + parse_claude_transcript_calls, + write_claude_mcp_config, +) +from evals.drivers.codex import ( + CodexCliDriver, + find_codex_rollout, + parse_codex_jsonl_events, + parse_codex_rollout_calls, + write_codex_mcp_override_args, +) +from evals.drivers.opencode import OpencodeCliDriver, write_opencode_mcp_config +from evals.drivers.process import kill_process_group, note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_wrap_server_command, + wait_for_proxy_meta, +) + +# Registry +# --------------------------------------------------------------------------- + +KNOWN_DRIVERS = frozenset({"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) + + +def get_driver(name: str, **kwargs: Any) -> AgentDriver | None: + """Return a driver instance, or None for the in-process ``sdk`` path.""" + key = (name or "sdk").strip().lower() + if key == "sdk": + return None # handled inline in evals.run + if key == "claude-cli": + return ClaudeCliDriver(**kwargs) + if key == "codex-cli": + return CodexCliDriver(**kwargs) + if key == "antigravity-cli": + return AntigravityCliDriver(**kwargs) + if key == "opencode-cli": + return OpencodeCliDriver(**kwargs) + raise ValueError(f"unknown driver {name!r}; expected one of {sorted(KNOWN_DRIVERS)}") + + +__all__ = [ + "KNOWN_DRIVERS", + "AgentDriver", + "AgentRun", + "AntigravityCliDriver", + "ClaudeCliDriver", + "CodexCliDriver", + "OpencodeCliDriver", + "REPO_ROOT", + "agent_run_to_harness_dict", + "apply_proxy_sidecar", + "ensure_proxy_pythonpath", + "find_claude_transcript", + "find_codex_rollout", + "get_driver", + "harvest_proxy_after_cli_timeout", + "is_plane_mcp_tool", + "kill_process_group", + "load_proxy_sidecar", + "load_proxy_sidecar_calls", + "normalize_claude_usage", + "normalize_tool_call", + "note_timeout_kill", + "parse_claude_json_result", + "parse_claude_transcript_calls", + "parse_codex_jsonl_events", + "parse_codex_rollout_calls", + "prepare_antigravity_fake_home", + "proxy_wrap_server_command", + "run_cli_subprocess", + "split_plane_and_client_calls", + "strip_mcp_prefix", + "wait_for_proxy_meta", + "write_antigravity_mcp_config", + "write_claude_mcp_config", + "write_codex_mcp_override_args", + "write_opencode_mcp_config", +] diff --git a/evals/drivers/antigravity.py b/evals/drivers/antigravity.py new file mode 100644 index 00000000..0fb0d4dc --- /dev/null +++ b/evals/drivers/antigravity.py @@ -0,0 +1,276 @@ +"""Antigravity CLI (agy) driver — proxy-first measurement.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.drivers.base import REPO_ROOT, AgentRun +from evals.drivers.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) + +# Antigravity CLI (agy) — proxy-first +# --------------------------------------------------------------------------- + + +def write_antigravity_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write ``mcpServers`` map JSON (Antigravity / agy mcp_config shape).""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def prepare_antigravity_fake_home( + fake_home: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + real_home: Path | None = None, +) -> None: + """Build an isolated HOME for agy with MCP config + shared auth artifacts. + + Writes mcp_config.json to BOTH documented locations (cheap; live probe + should settle which path agy actually reads): + - ~/.gemini/config/mcp_config.json + - ~/.gemini/antigravity-cli/mcp_config.json + + Creates ``antigravity-cli`` as a **real directory** (never a symlink of the + whole tree — that would write mcp_config and runtime logs into real user + state). Auth artifacts (``antigravity-oauth-token``) are plain **copies** — + never symlinks — so an in-place token refresh cannot write through into the + real home. Staleness over a single eval run is negligible. + """ + real_home = real_home or Path.home() + gemini_root = fake_home / ".gemini" + gemini_root.mkdir(parents=True, exist_ok=True) + + real_cli = real_home / ".gemini" / "antigravity-cli" + fake_cli = gemini_root / "antigravity-cli" + # Always a real directory — never symlink the whole tree. + if fake_cli.is_symlink() or fake_cli.is_file(): + fake_cli.unlink() + fake_cli.mkdir(parents=True, exist_ok=True) + + # Share auth via plain COPY only — never symlink (in-place token refresh + # must not write through into the real home). + if real_cli.is_dir(): + for name in ("antigravity-oauth-token",): + src = real_cli / name + dst = fake_cli / name + if src.is_file() and not dst.exists(): + try: + dst.write_bytes(src.read_bytes()) + except OSError: + pass + + # Dual write as real files (not through any symlink). + for rel in ( + Path(".gemini") / "config" / "mcp_config.json", + Path(".gemini") / "antigravity-cli" / "mcp_config.json", + ): + write_antigravity_mcp_config( + fake_home / rel, + command=command, + args=args, + env=env, + server_name="plane", + ) + + +class AntigravityCliDriver: + """Run tasks via Google Antigravity CLI (``agy``). + + Probed flags (2026-08-12, ``agy --help``): + - ``-p`` / ``--print`` headless single-prompt mode + - ``--output-format`` text|json|stream-json + - ``--model``, ``--dangerously-skip-permissions`` + - MCP via ``~/.gemini/config/mcp_config.json`` (``mcpServers`` map; + stdio: command/args/env). No CLI flag for MCP config → HOME isolation. + - No max-turns / turn-cap flag in help → ``hit_max_turns=False`` + note. + + Tool calls come from the recording proxy sidecar (protocol-layer), not + agy stdout parsing. + """ + + name = "antigravity-cli" + + def __init__( + self, + *, + agy_bin: str = "agy", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.agy_bin = agy_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = ["no_turn_cap"] + t0 = time.perf_counter() + child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + + with tempfile.TemporaryDirectory(prefix="plane-eval-antigravity-") as td: + td_path = Path(td) + sidecar = td_path / "proxy-sidecar.jsonl" + fake_home = td_path / "home" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env_plane = ensure_proxy_pythonpath(child_env_plane) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + prepare_antigravity_fake_home( + fake_home, + command=server_cmd, + args=server_args, + env=child_env_plane, + ) + + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd: list[str] = [ + self.agy_bin, + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + ] + if model: + cmd.extend(["--model", model]) + cmd.append(full_prompt) + + run_env = {**os.environ, "HOME": str(fake_home)} + if "PATH" in child_env_plane: + run_env["PATH"] = child_env_plane["PATH"] + + timeout_s = max(120, max_turns * 60) + try: + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + env=run_env, + ) + except TypeError: + # Some test runners reject ``env=``; retry without it. + # TimeoutExpired from this path must still hit the harvest below. + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + usage_scope="run", + call_source=call_source, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proc.returncode != 0: + notes.append(f"agy_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + final_text = stdout.strip() + try: + if final_text.lstrip().startswith("{"): + blob = json.loads(final_text) + if isinstance(blob, dict): + final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) + except json.JSONDecodeError: + pass + + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=final_text, + usage=None, + stopped_reason="error" if proc.returncode else "end_turn", + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +__all__ = [ + "AntigravityCliDriver", + "prepare_antigravity_fake_home", + "write_antigravity_mcp_config", +] diff --git a/evals/drivers/base.py b/evals/drivers/base.py new file mode 100644 index 00000000..3ff0bed5 --- /dev/null +++ b/evals/drivers/base.py @@ -0,0 +1,276 @@ +"""Shared types and tool-name helpers for eval agent drivers.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# mcp__plane__list_work_items → list_work_items +# mcp__plane-mcp-server__foo → foo +_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") +# Alternate: mcp__server__tool with multi-segment server names +_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") + + +@dataclass +class AgentRun: + """Normalized result of one agent task execution.""" + + # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} + calls: list[dict[str, Any]] + final_text: str + usage: dict[str, Any] | None + stopped_reason: str + raw_ref: str | None = None + # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens + usage_total: dict[str, Any] | None = None + # Harness extras (optional; defaults keep SDK path simple) + usage_scope: str = "run" # 'run' | 'iteration' + call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'sdk' + hit_max_turns: bool = False + wall_time_s: float = 0.0 + experimental: bool = False + notes: list[str] = field(default_factory=list) + + +class AgentDriver(Protocol): + """Pluggable agent backend for evals.run.""" + + name: str + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: ... + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def strip_mcp_prefix(name: str) -> str: + """Strip Claude/Codex MCP tool name prefixes for classification. + + Examples: + mcp__plane__list_work_items → list_work_items + mcp__plane-mcp-server__find_work_items → find_work_items + """ + if not name: + return name + m = _MCP_PREFIX_RE2.match(name) + if m: + return m.group(1) + return name + + +def is_plane_mcp_tool(name: str) -> bool: + """True when the raw tool name is from our Plane MCP server (pre-strip). + + Claude surfaces MCP tools as ``mcp____``. Our config registers + the server as ``plane``, so names look like ``mcp__plane__find_work_items``. + Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. + """ + if not name: + return False + # mcp__plane__tool or mcp__plane-foo__tool + return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") + + +def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: + """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" + raw = str(name or "") + if not isinstance(args, dict): + args = {"_raw": args} + if is_plane_mcp_tool(raw): + return { + "tool": strip_mcp_prefix(raw), + "args": args, + "origin": "plane", + "raw_tool": raw, + } + return { + "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) + "args": args, + "origin": "client", + "raw_tool": raw, + } + + +def split_plane_and_client_calls( + calls: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Partition tagged calls into plane vs client lists. + + Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls + (SDK path) default to plane so existing harness behavior is unchanged. + """ + plane: list[dict[str, Any]] = [] + client: list[dict[str, Any]] = [] + for c in calls: + origin = c.get("origin") + if origin is None: + raw = str(c.get("raw_tool") or c.get("tool") or "") + if is_plane_mcp_tool(raw): + origin = "plane" + elif raw.startswith("mcp__"): + origin = "client" # other MCP server + else: + origin = "plane" # bare name → assume plane (SDK) + if origin == "client": + client.append(c) + else: + plane.append(c) + return plane, client + + +def agent_run_to_harness_dict( + run: AgentRun, + *, + optimal: set[str], + alternate: set[str], + classify: Callable[[str, set[str], set[str]], str], + skip_result_tokens: bool = True, +) -> dict[str, Any]: + """Map an ``AgentRun`` onto the dict shape expected by ``run_live`` rows. + + Only **plane** MCP tools are classified and counted in ``num_calls`` / + mispick metrics. Client built-ins (``ToolSearch``, …) go to + ``client_tool_calls`` and are excluded. + + CLI drivers never populate ``cum_input_tokens`` from bare + ``usage.input_tokens`` (that field is uncached-only under Claude Code and + misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. + """ + # Re-split in case callers passed a mixed list + plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) + client_src = list(run.client_tool_calls) + client_extra + + calls: list[dict[str, Any]] = [] + for c in plane_src: + tool = c.get("tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + rec: dict[str, Any] = { + "tool": tool, + "class": classify(str(tool), optimal, alternate), + "args_chars": args_chars, + "result_tokens": None, + "result_chars": int(c["result_chars"]) if c.get("result_chars") is not None else 0, + "result_kind": "text", + "is_error": bool(c.get("is_error")), + } + if c.get("duration_ms") is not None: + rec["duration_ms"] = c["duration_ms"] + # Action-dispatch surfaces: the action arg IS the second half of the + # tool choice — keep it (args content is otherwise not persisted). + if isinstance(args, dict) and isinstance(args.get("action"), str): + rec["action"] = args["action"] + if skip_result_tokens: + rec["result_tokens_skipped"] = "no API key / CLI driver has no count_tokens" + calls.append(rec) + + client_tool_calls: list[dict[str, Any]] = [] + for c in client_src: + tool = c.get("tool") or c.get("raw_tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + client_tool_calls.append( + { + "tool": tool, + "args_chars": args_chars, + "raw_tool": c.get("raw_tool") or tool, + } + ) + + stop_reason = run.stopped_reason + hit_max = run.hit_max_turns + if hit_max: + stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" + + errored = sum(1 for c in calls if c.get("is_error")) + alternate_n = sum(1 for c in calls if c["class"] == "alternate") + out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") + + # CLI path: never write misleading cum_input_tokens from uncached-only field. + # usage_total is driver-owned — do not re-derive it here (Claude vs Codex + # shapes differ; a generic Claude rebuild mislabels other vendors). + is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" + usage_total = run.usage_total + + if is_cli and skip_result_tokens: + cum_input: int | None = None + cum_reason: str | None = ( + "CLI driver: Claude usage.input_tokens is uncached-only; " + "see usage_total (cache_read/cache_creation/output/cost) for run accounting" + ) + usage_per_iteration: list[dict[str, int]] = [] + else: + cum_input = 0 + cum_reason = None + usage_per_iteration = [] + if run.usage and run.usage_scope == "iteration": + pass # SDK fills this separately + + return { + "final_text": run.final_text, + "calls": calls, + "num_calls": len(calls), + "client_tool_calls": client_tool_calls, + "client_tool_call_count": len(client_tool_calls), + "errored_calls": errored, + "alternate_calls": alternate_n, + "out_of_set_calls": out_of_set_n, + "total_result_tokens": 0 + if skip_result_tokens + else sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None), + "usage_per_iteration": usage_per_iteration, + "cum_input_tokens": cum_input, + "cum_input_tokens_reason": cum_reason, + "wall_time_s": run.wall_time_s, + "stop_reason": stop_reason, + "hit_max_iterations": hit_max, + "result_pair_mismatch": False, + "token_count_failures": 0, + "usage_scope": run.usage_scope, + "call_source": run.call_source, + "driver_raw_ref": run.raw_ref, + "driver_notes": list(run.notes), + "result_tokens_skipped_reason": ( + "CLI driver: count_tokens requires Anthropic API key; skipped" if skip_result_tokens else None + ), + "usage": run.usage, + "usage_total": usage_total, + } + + +__all__ = [ + "REPO_ROOT", + "AgentRun", + "AgentDriver", + "agent_run_to_harness_dict", + "is_plane_mcp_tool", + "normalize_tool_call", + "split_plane_and_client_calls", + "strip_mcp_prefix", +] diff --git a/evals/drivers/claude.py b/evals/drivers/claude.py new file mode 100644 index 00000000..6e7a0a29 --- /dev/null +++ b/evals/drivers/claude.py @@ -0,0 +1,495 @@ +"""Claude Code CLI driver and transcript/JSON parsers.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.drivers.base import ( + REPO_ROOT, + AgentRun, + normalize_tool_call, + split_plane_and_client_calls, +) +from evals.drivers.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) + + +def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Parse Claude print-mode usage into (raw_usage, usage_total). + + Real envelope (probed 2026-08-12, ``claude -p --output-format json``):: + + { + "usage": { + "input_tokens": 10, # uncached NEW input only — NOT run total + "cache_creation_input_tokens": 17459, + "cache_read_input_tokens": 18464, + "output_tokens": 143, + "iterations": [...], + ... + }, + "modelUsage": { + "": { + "inputTokens": 10, + "outputTokens": 143, + "cacheReadInputTokens": 18464, + "cacheCreationInputTokens": 17459, + "costUSD": 0.037, + ... + } + }, + "total_cost_usd": 0.037 + } + + ``usage.input_tokens`` alone is misleading for multi-turn cached runs (live + rows showed 8–10 while cache_read was 180k+). We keep the split fields and + compute an inclusive total under ``usage_total``; callers must **not** copy + bare ``input_tokens`` into ``cum_input_tokens``. + """ + usage = data.get("usage") + if usage is not None and not isinstance(usage, dict): + usage = None + model_usage = data.get("modelUsage") or data.get("model_usage") + if model_usage is not None and not isinstance(model_usage, dict): + model_usage = None + cost = data.get("total_cost_usd") + if cost is None and isinstance(usage, dict): + cost = usage.get("total_cost_usd") + + if usage is None and model_usage is None and cost is None: + return None, None + + raw = dict(usage or {}) + if cost is not None: + raw["total_cost_usd"] = cost + if model_usage is not None: + raw["modelUsage"] = model_usage + + # Prefer summing modelUsage (per-model run totals) when present + sum_in = sum_out = sum_cr = sum_cc = sum_cost = 0.0 + used_model_usage = False + if model_usage: + for _mid, mu in model_usage.items(): + if not isinstance(mu, dict): + continue + used_model_usage = True + sum_in += float(mu.get("inputTokens") or mu.get("input_tokens") or 0) + sum_out += float(mu.get("outputTokens") or mu.get("output_tokens") or 0) + sum_cr += float(mu.get("cacheReadInputTokens") or mu.get("cache_read_input_tokens") or 0) + sum_cc += float(mu.get("cacheCreationInputTokens") or mu.get("cache_creation_input_tokens") or 0) + sum_cost += float(mu.get("costUSD") or mu.get("cost_usd") or 0) + + if used_model_usage: + uncached_in = int(sum_in) + out_tok = int(sum_out) + cache_read = int(sum_cr) + cache_write = int(sum_cc) + total_cost = float(sum_cost) if sum_cost else cost + else: + uncached_in = int(raw.get("input_tokens") or 0) + out_tok = int(raw.get("output_tokens") or 0) + cache_read = int(raw.get("cache_read_input_tokens") or 0) + cache_write = int(raw.get("cache_creation_input_tokens") or 0) + total_cost = cost + + usage_total: dict[str, Any] = { + "input_tokens": uncached_in, # uncached / new tokens only + "output_tokens": out_tok, + "cache_read_input_tokens": cache_read, + "cache_creation_input_tokens": cache_write, + "total_input_tokens_including_cache": uncached_in + cache_read + cache_write, + "total_cost_usd": total_cost, + "modelUsage": model_usage, + "source": "modelUsage" if used_model_usage else "usage", + } + return raw, usage_total + + +def _claude_project_dir(cwd: Path) -> Path: + """Map a cwd to ``~/.claude/projects/`` (``/`` → ``-``).""" + munged = str(cwd.resolve()).replace("/", "-") + return Path.home() / ".claude" / "projects" / munged + + +def parse_claude_json_result(payload: dict[str, Any] | str) -> dict[str, Any]: + """Extract final text, usage, session id, num_turns from ``claude -p --output-format json``. + + The print-mode JSON envelope is a single object (``type=result``) with + ``result``, ``session_id``, ``num_turns``, ``total_cost_usd``, ``usage``, + and ``modelUsage``. Per-call tool detail is usually **absent** — callers + should fall back to the session transcript. + """ + if isinstance(payload, str): + payload = json.loads(payload) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object from claude, got {type(payload)}") + + data = payload + + final = data.get("result") + if final is None: + final = data.get("final_text") or data.get("text") or "" + if not isinstance(final, str): + final = json.dumps(final, default=str) + + usage, usage_total = normalize_claude_usage(data) + + session_id = data.get("session_id") or data.get("sessionId") + num_turns = data.get("num_turns") + if num_turns is None: + num_turns = data.get("numTurns") + is_error = bool(data.get("is_error") or data.get("isError")) + subtype = data.get("subtype") or "" + stop_reason = data.get("stop_reason") or data.get("terminal_reason") or "" + + # Tool calls rarely present in the result envelope; collect if present. + calls: list[dict[str, Any]] = [] + for key in ("tool_calls", "tools", "calls"): + raw = data.get(key) + if isinstance(raw, list): + for item in raw: + if not isinstance(item, dict): + continue + name = item.get("name") or item.get("tool") or "" + args = item.get("input") or item.get("arguments") or item.get("args") or {} + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + args = {"_raw": args} + calls.append(normalize_tool_call(str(name), args)) + + # Preserve Claude error subtypes (e.g. error_during_execution, error_max_turns). + # is_error alone collapses to "error" and loses the subtype run.py uses for infra_cli. + if is_error and subtype and str(subtype) not in ("success", ""): + stopped = str(subtype) + elif is_error: + stopped = "error" + else: + stopped = str(stop_reason) if stop_reason else "end_turn" + if subtype and subtype not in ("success", "") and stopped == "end_turn": + stopped = str(subtype) + + plane_calls, client_calls = split_plane_and_client_calls(calls) + + return { + "final_text": final, + "usage": usage, + "usage_total": usage_total, + "session_id": session_id, + "num_turns": int(num_turns) if num_turns is not None else None, + "calls": plane_calls, + "client_tool_calls": client_calls, + "stopped_reason": stopped, + "raw": data, + } + + +def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]]: + """Parse ``tool_use`` blocks from a Claude Code session JSONL transcript. + + Returns tagged calls (``origin`` plane|client). Use + ``split_plane_and_client_calls`` before classification. + """ + calls: list[dict[str, Any]] = [] + if not transcript_path.is_file(): + return calls + with transcript_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + msg = row.get("message") if isinstance(row, dict) else None + if not isinstance(msg, dict): + if row.get("type") == "assistant" and isinstance(row.get("content"), list): + content = row["content"] + else: + continue + else: + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") != "tool_use": + continue + name = str(block.get("name") or "") + args = block.get("input") or {} + if not isinstance(args, dict): + args = {"_raw": args} + calls.append(normalize_tool_call(name, args)) + return calls + + +def find_claude_transcript(session_id: str | None, cwd: Path) -> Path | None: + """Locate ``~/.claude/projects//.jsonl``.""" + if not session_id: + return None + candidate = _claude_project_dir(cwd) / f"{session_id}.jsonl" + if candidate.is_file(): + return candidate + # Fallback: scan project dir for a file containing the session id + proj = _claude_project_dir(cwd) + if not proj.is_dir(): + return None + direct = proj / f"{session_id}.jsonl" + if direct.is_file(): + return direct + for p in proj.glob("*.jsonl"): + if session_id in p.name: + return p + return None + + +def write_claude_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write a Claude-compatible mcp-config JSON file.""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Claude CLI driver +# --------------------------------------------------------------------------- + + +class ClaudeCliDriver: + """Run tasks via ``claude -p`` on the user's Claude Code subscription.""" + + name = "claude-cli" + + def __init__( + self, + *, + claude_bin: str = "claude", + python_bin: str | None = None, + permission_mode: str = "bypassPermissions", + strict_mcp: bool = True, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.claude_bin = claude_bin + self.python_bin = python_bin or sys.executable + self.permission_mode = permission_mode + self.strict_mcp = strict_mcp + self._runner = runner or run_cli_subprocess + # Full replacement for the MCP server launch (external surfaces under + # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = [] + t0 = time.perf_counter() + + with tempfile.TemporaryDirectory(prefix="plane-eval-claude-") as td: + td_path = Path(td) + mcp_cfg = td_path / "mcp.json" + sidecar = td_path / "proxy-sidecar.jsonl" + # Only pass Plane-related env into the MCP child (plus PATH/HOME if present). + child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env = ensure_proxy_pythonpath(child_env) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + write_claude_mcp_config( + mcp_cfg, + command=server_cmd, + args=server_args, + env=child_env, + server_name="plane", + ) + + cmd: list[str] = [ + self.claude_bin, + "-p", + "--output-format", + "json", + "--mcp-config", + str(mcp_cfg), + "--permission-mode", + self.permission_mode, + "--max-turns", + str(max_turns), + ] + if self.strict_mcp: + cmd.append("--strict-mcp-config") + if model: + cmd.extend(["--model", model]) + if system: + cmd.extend(["--append-system-prompt", system]) + # Allow MCP tools from our server without interactive prompts + # --allowedTools is variadic and would swallow the trailing prompt; use = form. + cmd.append("--allowedTools=mcp__plane__*") + cmd.append(prompt) + + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + # Wait for proxy finalization before harvesting / temp dir teardown. + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = harvest_proxy_after_cli_timeout( + calls, client_calls, sidecar, notes + ) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + + parsed: dict[str, Any] | None = None + parse_err: str | None = None + # JSON may be the whole stdout or the last JSON object line + for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): + if not candidate or not candidate.lstrip().startswith("{"): + continue + try: + parsed = parse_claude_json_result(candidate) + break + except (json.JSONDecodeError, ValueError, TypeError) as exc: + parse_err = str(exc) + continue + + if parsed is None: + notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + if self.use_proxy: + apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise RuntimeError(f"claude cli failed: {detail}") + + # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + # JSON rarely embeds per-call tool detail — prefer transcript when present. + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "json" if (calls or client_calls) else "json" + session_id = parsed.get("session_id") + transcript = find_claude_transcript(session_id, cwd) + if transcript is not None: + tagged = parse_claude_transcript_calls(transcript) + t_plane, t_client = split_plane_and_client_calls(tagged) + if t_plane or t_client: + calls, client_calls = t_plane, t_client + call_source = "transcript" + notes.append(f"calls_from_transcript:{transcript}") + if not calls and not client_calls: + notes.append("no_tool_calls_in_json_or_transcript") + + # Proxy sidecar (when enabled) replaces CLI-parsed plane calls. + if self.use_proxy: + calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proxy_src == "proxy": + call_source = "proxy" + + num_turns = parsed.get("num_turns") + hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) + stopped = parsed["stopped_reason"] + if hit_max and stopped in ("end_turn", "completed", ""): + stopped = "max_turns" + + raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=parsed["final_text"], + usage=parsed.get("usage"), + usage_total=parsed.get("usage_total"), + stopped_reason=stopped, + raw_ref=raw_ref, + usage_scope="run", + call_source=call_source, + hit_max_turns=hit_max, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +__all__ = [ + "ClaudeCliDriver", + "find_claude_transcript", + "normalize_claude_usage", + "parse_claude_json_result", + "parse_claude_transcript_calls", + "write_claude_mcp_config", +] diff --git a/evals/drivers/codex.py b/evals/drivers/codex.py new file mode 100644 index 00000000..01ffa440 --- /dev/null +++ b/evals/drivers/codex.py @@ -0,0 +1,431 @@ +"""Codex CLI driver and JSONL/rollout parsers.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.drivers.base import ( + REPO_ROOT, + AgentRun, + normalize_tool_call, + split_plane_and_client_calls, +) +from evals.drivers.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) + + +def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + return {"_raw": raw_args} + return args if isinstance(args, dict) else {"_raw": args} + if isinstance(raw_args, dict): + return raw_args + return {"_raw": raw_args} + + +def parse_codex_jsonl_events(lines: list[str] | str) -> dict[str, Any]: + """Parse ``codex exec --json`` stdout (JSONL) for function_call + final text + usage. + + Supports both schemas: + - **v0.147+ streamable**: ``thread.started`` / ``item.completed`` / ``turn.completed`` + (``thread_id`` matches rollout filename suffix). + - **Legacy**: ``session_meta`` / ``response_item`` / ``event_msg`` payloads. + New keys are tried first; legacy handling is retained. + """ + if isinstance(lines, str): + lines = lines.splitlines() + calls: list[dict[str, Any]] = [] + final_parts: list[str] = [] + usage: dict[str, Any] | None = None + stopped = "end_turn" + session_id: str | None = None + + for line in lines: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + rtype = row.get("type") + + # --- New schema (codex exec --json v0.147+): try first --- + if rtype == "thread.started": + session_id = row.get("thread_id") or session_id + continue + if rtype == "turn.started": + continue + if rtype == "item.completed": + item = row.get("item") if isinstance(row.get("item"), dict) else {} + itype = item.get("type") + if itype == "agent_message": + text = item.get("text") + if text: + final_parts.append(str(text)) + elif itype in ( + "function_call", + "tool_call", + "mcp_tool_call", + "command_execution", + "file_change", + ): + # Proxy sidecar is primary for plane calls; harvest best-effort names. + name = str(item.get("name") or item.get("tool") or item.get("command") or itype) + args = item.get("arguments") or item.get("args") or item.get("input") or {} + calls.append(normalize_tool_call(name, _codex_parse_tool_args(args))) + continue + if rtype == "turn.completed": + u = row.get("usage") if isinstance(row.get("usage"), dict) else {} + if u: + usage = { + "input_tokens": u.get("input_tokens", 0) or 0, + "output_tokens": u.get("output_tokens", 0) or 0, + "cache_read_input_tokens": u.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": u.get("cache_write_input_tokens", 0) or 0, + "total_tokens": u.get("total_tokens"), + } + continue + + # --- Legacy schema --- + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + + if rtype == "session_meta": + session_id = payload.get("id") or session_id + continue + + if rtype == "response_item": + pt = payload.get("type") + if pt == "function_call": + name = str(payload.get("name") or "") + calls.append(normalize_tool_call(name, _codex_parse_tool_args(payload.get("arguments") or "{}"))) + elif pt == "message": + # Assistant final-ish content + role = payload.get("role") + content = payload.get("content") + if role == "assistant" and isinstance(content, list): + for c in content: + if isinstance(c, dict) and c.get("type") in ("output_text", "text"): + t = c.get("text") or c.get("output_text") + if t: + final_parts.append(str(t)) + continue + + if rtype == "event_msg": + pt = payload.get("type") + if pt == "agent_message": + msg = payload.get("message") or payload.get("text") + if msg: + final_parts.append(str(msg)) + elif pt == "token_count": + info = payload.get("info") or {} + total = info.get("total_token_usage") or info.get("last_token_usage") or {} + if isinstance(total, dict): + usage = { + "input_tokens": total.get("input_tokens", 0) or 0, + "output_tokens": total.get("output_tokens", 0) or 0, + "cache_read_input_tokens": total.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": total.get("cache_write_input_tokens", 0) or 0, + "total_tokens": total.get("total_tokens"), + } + elif pt == "task_complete": + stopped = "end_turn" + elif pt == "turn_aborted": + stopped = "aborted" + continue + + plane_calls, client_calls = split_plane_and_client_calls(calls) + return { + "calls": plane_calls, + "client_tool_calls": client_calls, + "final_text": "\n".join(final_parts).strip(), + "usage": usage, + "stopped_reason": stopped, + "session_id": session_id, + } + + +def parse_codex_rollout_calls(rollout_path: Path) -> list[dict[str, Any]]: + """Parse function_call records from a Codex session rollout JSONL (plane only).""" + if not rollout_path.is_file(): + return [] + lines = rollout_path.read_text(encoding="utf-8").splitlines() + return parse_codex_jsonl_events(lines)["calls"] + + +def find_codex_rollout(session_id: str | None, *, after_ts: float | None = None) -> Path | None: + """Find a rollout JSONL under ``~/.codex/sessions`` matching *session_id* exactly. + + Matches filename containing the id (rollout filenames end with ``thread_id``) + or a first-line ``session_meta`` / ``thread.started`` id field. + + **No newest-after-ts fallback**: under parallel runs that would pick another + task's rollout and corrupt final_text. Callers should note + ``codex_rollout_unmatched`` when this returns None. + + ``after_ts`` is accepted for API compatibility but ignored. + """ + del after_ts # intentionally unused — see docstring + if not session_id: + return None + root = Path.home() / ".codex" / "sessions" + if not root.is_dir(): + return None + sid = str(session_id) + for p in root.rglob("*.jsonl"): + if sid in p.name: + return p + for p in root.rglob("rollout-*.jsonl"): + try: + with p.open(encoding="utf-8") as fh: + first = fh.readline() + row = json.loads(first) + except Exception: + continue + # Legacy session_meta + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + if payload.get("id") == sid: + return p + # New thread.started on first line (unusual but possible) + if row.get("type") == "thread.started" and row.get("thread_id") == sid: + return p + if row.get("thread_id") == sid or row.get("session_id") == sid: + return p + return None + + +def write_codex_mcp_override_args( + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> list[str]: + """Build ``codex exec -c ...`` overrides for a stdio MCP server. + + Codex stores MCP under ``[mcp_servers.]`` with ``command``, ``args``, + and ``env`` (see user config.toml). Overrides use dotted ``-c`` paths. + """ + out: list[str] = [ + "-c", + f"mcp_servers.{server_name}.command={json.dumps(command)}", + "-c", + f"mcp_servers.{server_name}.args={json.dumps(args)}", + ] + # env table — pass each key + for k, v in env.items(): + out.extend(["-c", f"mcp_servers.{server_name}.env.{k}={json.dumps(v)}"]) + return out + + +# Codex CLI driver (experimental — do not spend live quota from CI) +# --------------------------------------------------------------------------- + + +class CodexCliDriver: + """Run tasks via ``codex exec`` (experimental; metered quota). + + Live invocation is supported for the interface, but the eval harness should + only exercise this driver when the team explicitly opts in. Offline tests + inject a fake runner and never touch the real binary. + """ + + name = "codex-cli" + experimental = True + + def __init__( + self, + *, + codex_bin: str = "codex", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + allow_live: bool = False, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.codex_bin = codex_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.allow_live = allow_live + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + cwd = (cwd or REPO_ROOT).resolve() + notes = ["experimental:codex-cli"] + if self._runner is run_cli_subprocess and not self.allow_live: + raise RuntimeError( + "CodexCliDriver refuses live runs by default (metered weekly quota). " + "Pass allow_live=True or inject a fake runner for tests." + ) + + child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + with tempfile.TemporaryDirectory(prefix="plane-eval-codex-") as td: + td_path = Path(td) + sidecar = td_path / "proxy-sidecar.jsonl" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + server_cmd, server_args = wrapped[0], wrapped[1:] + child_env = ensure_proxy_pythonpath(child_env) + else: + server_cmd, server_args = real_cmd[0], real_cmd[1:] + mcp_args = write_codex_mcp_override_args( + command=server_cmd, + args=server_args, + env=child_env, + server_name="plane", + ) + cmd: list[str] = [ + self.codex_bin, + "exec", + "--json", + "--skip-git-repo-check", + *mcp_args, + ] + if model: + cmd.extend(["-m", model]) + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd.append(full_prompt) + + t0 = time.perf_counter() + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "stream" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + experimental=True, + notes=notes, + ) + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + parsed = parse_codex_jsonl_events(stdout) + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "stream" + session_id = parsed.get("session_id") + + # Exact-match rollout only — never steal another parallel task's file. + need_rollout = (not calls and not client_calls) or not parsed.get("final_text") + if need_rollout and session_id: + rollout = find_codex_rollout(session_id) + if rollout is not None: + full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) + if not calls and not client_calls: + calls = list(full.get("calls") or []) + client_calls = list(full.get("client_tool_calls") or []) + if calls or client_calls: + call_source = "transcript" + notes.append(f"calls_from_rollout:{rollout}") + if full.get("final_text") and not parsed.get("final_text"): + parsed["final_text"] = full["final_text"] + notes.append(f"final_text_from_rollout:{rollout}") + if full.get("usage") and not parsed.get("usage"): + parsed["usage"] = full["usage"] + else: + notes.append("codex_rollout_unmatched") + elif need_rollout and not session_id: + notes.append("codex_rollout_unmatched") + + if self.use_proxy: + calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proxy_src == "proxy": + call_source = "proxy" + + if proc.returncode != 0: + notes.append(f"codex_exit={proc.returncode}") + + usage = parsed.get("usage") + usage_total = None + if isinstance(usage, dict): + usage_total = { + "input_tokens": usage.get("input_tokens"), + "output_tokens": usage.get("output_tokens"), + "cache_read_input_tokens": usage.get("cache_read_input_tokens"), + "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), + "total_input_tokens_including_cache": ( + int(usage.get("input_tokens") or 0) + + int(usage.get("cache_read_input_tokens") or 0) + + int(usage.get("cache_creation_input_tokens") or 0) + ), + "source": "codex_token_count", + } + + raw_ref = f"session:{session_id}" if session_id else None + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=parsed.get("final_text") or "", + usage=usage, + usage_total=usage_total, + stopped_reason=parsed.get("stopped_reason") or "end_turn", + raw_ref=raw_ref, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, # codex exec has no max-turns flag in --help + wall_time_s=round(wall, 3), + experimental=True, + notes=notes, + ) + + +__all__ = [ + "CodexCliDriver", + "find_codex_rollout", + "parse_codex_jsonl_events", + "parse_codex_rollout_calls", + "write_codex_mcp_override_args", +] diff --git a/evals/drivers/opencode.py b/evals/drivers/opencode.py new file mode 100644 index 00000000..17a44a97 --- /dev/null +++ b/evals/drivers/opencode.py @@ -0,0 +1,212 @@ +"""OpenCode CLI driver — proxy-first measurement.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.drivers.base import REPO_ROOT, AgentRun +from evals.drivers.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) + +# OpenCode CLI — proxy-first +# --------------------------------------------------------------------------- + + +def write_opencode_mcp_config( + path: Path, + *, + command: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write project ``opencode.json`` with a local MCP server entry. + + Schema (opencode.ai docs / probed binary strings, 2026-08-12):: + + {"mcp": {"plane": {"type": "local", "command": [...], "environment": {...}}}} + """ + cfg = { + "$schema": "https://opencode.ai/config.json", + "mcp": { + server_name: { + "type": "local", + "command": list(command), + "environment": env, + "enabled": True, + } + }, + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +class OpencodeCliDriver: + """Run tasks via ``opencode run`` (proxy-first call recording). + + Probed flags (2026-08-12): + - ``opencode run [message..]`` non-interactive + - ``--format json|default``, ``-m/--model`` + - MCP via ``opencode.json`` ``mcp`` section (local: type/command/environment) + written into the task cwd (or a temp project dir). + - No turn-cap flag → ``hit_max_turns=False`` + note ``no_turn_cap``. + """ + + name = "opencode-cli" + + def __init__( + self, + *, + opencode_bin: str = "opencode", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + ) -> None: + self.opencode_bin = opencode_bin + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + base_cwd = (cwd or REPO_ROOT).resolve() + notes: list[str] = ["no_turn_cap"] + t0 = time.perf_counter() + child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} + + with tempfile.TemporaryDirectory(prefix="plane-eval-opencode-", dir=str(base_cwd)) as td: + # Project-local opencode.json so we do not pollute the user's global config. + proj = Path(td) + sidecar = proj / "proxy-sidecar.jsonl" + if self.server_command: + real_cmd = list(self.server_command) + else: + real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] + if self.use_proxy: + launch = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + child_env_plane = ensure_proxy_pythonpath(child_env_plane) + else: + launch = real_cmd + write_opencode_mcp_config( + proj / "opencode.json", + command=launch, + env=child_env_plane, + server_name="plane", + ) + + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + cmd: list[str] = [ + self.opencode_bin, + "run", + "--format", + "json", + ] + if model: + cmd.extend(["-m", model]) + cmd.append(full_prompt) + + timeout_s = max(120, max_turns * 60) + try: + proc = self._runner( + cmd, + cwd=str(proj), + capture_output=True, + text=True, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - t0 + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls_to: list[dict[str, Any]] = [] + client_to: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( + calls_to, client_to, sidecar, notes + ) + return AgentRun( + calls=calls_to, + client_tool_calls=client_to, + final_text="", + usage=None, + stopped_reason="timeout", + usage_scope="run", + call_source=call_source, + wall_time_s=round(wall, 3), + notes=notes, + ) + + wall = time.perf_counter() - t0 + stdout = proc.stdout or "" + stderr = proc.stderr or "" + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = "json" + if self.use_proxy: + calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) + if proc.returncode != 0: + notes.append(f"opencode_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + final_text = stdout.strip() + # JSONL events: concatenate text-ish fields best-effort. + if final_text and "\n" in final_text: + parts: list[str] = [] + for line in final_text.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + for key in ("text", "message", "part", "delta"): + v = row.get(key) + if isinstance(v, str) and v.strip(): + parts.append(v) + if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): + parts.append(row["content"]) + if parts: + final_text = "\n".join(parts) + + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text=final_text, + usage=None, + stopped_reason="error" if proc.returncode else "end_turn", + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + notes=notes, + ) + + +__all__ = [ + "OpencodeCliDriver", + "write_opencode_mcp_config", +] diff --git a/evals/drivers/process.py b/evals/drivers/process.py new file mode 100644 index 00000000..f092cfe1 --- /dev/null +++ b/evals/drivers/process.py @@ -0,0 +1,135 @@ +"""CLI subprocess lifecycle: process-group launch, timeout kill, bounded reap.""" + +from __future__ import annotations + +import os +import signal +import subprocess +from typing import Any + +# Bounded drain after process-group kill so communicate() never hangs forever +# when a grandchild still holds the pipe open. +_CLI_TIMEOUT_DRAIN_S = 2.0 + + +def kill_process_group(proc: subprocess.Popen[Any]) -> bool: + """SIGKILL the process group whose leader is ``proc``. + + With ``start_new_session=True``, ``pgid == proc.pid`` even after the leader + has been reaped — call ``killpg(proc.pid, …)`` directly (never fall back to + killing only the leader, which leaves grandchildren alive). + + Returns True if ``killpg`` delivered the signal; False if the group is + already fully gone (``ProcessLookupError`` = success for cleanup, but the + kill itself did not run). + """ + if proc.pid is None: + return False + try: + # Do NOT use getpgid: if the leader is already reaped, getpgid fails and + # a proc.kill() fallback would recreate the original orphan bug. + os.killpg(proc.pid, signal.SIGKILL) + return True + except ProcessLookupError: + # No process left in the group — fully gone. + return False + + +def _decode_pipe(data: str | bytes | None, *, text: bool) -> str | bytes | None: + if data is None or not text or isinstance(data, str): + return data + return data.decode("utf-8", errors="replace") + + +def _close_pipes_and_reap(proc: subprocess.Popen[Any], *, drain_s: float = _CLI_TIMEOUT_DRAIN_S) -> tuple[Any, Any]: + """Bounded drain / close after a group kill. Never hangs unbounded.""" + try: + return proc.communicate(timeout=drain_s) + except (subprocess.TimeoutExpired, ValueError, OSError): + for stream in (proc.stdout, proc.stderr): + if stream is not None: + try: + stream.close() + except Exception: + pass + try: + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + pass + return None, None + + +def run_cli_subprocess( + cmd: list[str], + *, + timeout: float | None = None, + cwd: str | None = None, + env: dict[str, str] | None = None, + capture_output: bool = True, + text: bool = True, + **_kwargs: Any, +) -> subprocess.CompletedProcess[Any]: + """Run a CLI in its own process group; kill the **whole group** on timeout/interrupt. + + Node wrappers (e.g. ``codex``) spawn native grandchildren. Plain + ``subprocess.run`` on timeout only kills the parent; the grandchild keeps + stdout open and ``communicate()`` hangs indefinitely. This runner: + + 1. launches with ``start_new_session=True`` (new process group; pgid=pid); + 2. on timeout **or any other exception** (incl. KeyboardInterrupt), + ``os.killpg(pid, SIGKILL)`` the group; + 3. drains pipes with a **bounded** second ``communicate`` (never unbounded). + + Raises ``subprocess.TimeoutExpired`` with attribute + ``killed_process_group=True`` only when killpg actually delivered the signal. + """ + popen_kwargs: dict[str, Any] = { + "cwd": cwd, + "start_new_session": True, + "stdout": subprocess.PIPE if capture_output else None, + "stderr": subprocess.PIPE if capture_output else None, + "text": text, + } + if env is not None: + popen_kwargs["env"] = env + + proc = subprocess.Popen(cmd, **popen_kwargs) # noqa: S603 — eval harness launches user CLIs + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + killed = kill_process_group(proc) + out, err = _close_pipes_and_reap(proc) + if out is None and err is None: + stdout = _decode_pipe(exc.stdout, text=text) or ("" if text else b"") + stderr = _decode_pipe(exc.stderr, text=text) or ("" if text else b"") + else: + stdout, stderr = out, err + te = subprocess.TimeoutExpired( + cmd=cmd, + timeout=timeout if timeout is not None else 0, + output=stdout, + stderr=stderr, + ) + te.killed_process_group = killed # type: ignore[attr-defined] + raise te from None + except BaseException: + # KeyboardInterrupt / SystemExit / etc. — do not leave the CLI tree running. + # start_new_session means SIGINT no longer reaches the group automatically. + kill_process_group(proc) + _close_pipes_and_reap(proc) + raise + + return subprocess.CompletedProcess(cmd, proc.returncode if proc.returncode is not None else 0, stdout, stderr) + + +def note_timeout_kill(notes: list[str], exc: BaseException) -> None: + """Append process-group kill note when killpg actually delivered the signal.""" + if getattr(exc, "killed_process_group", False): + notes.append("timeout_killed_process_group") + + +__all__ = [ + "kill_process_group", + "note_timeout_kill", + "run_cli_subprocess", +] diff --git a/evals/drivers/sidecar.py b/evals/drivers/sidecar.py new file mode 100644 index 00000000..6bb60bcb --- /dev/null +++ b/evals/drivers/sidecar.py @@ -0,0 +1,233 @@ +"""Recording-proxy glue: wrap commands, PYTHONPATH, load/harvest sidecar JSONL.""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +from evals.drivers.base import REPO_ROOT + + +def proxy_wrap_server_command( + real_command: list[str], + *, + sidecar_path: Path, + python_bin: str | None = None, +) -> list[str]: + """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" + py = python_bin or sys.executable + return [py, "-m", "evals.proxy", "--log", str(sidecar_path), "--", *real_command] + + +def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: + """Inject the repo root into PYTHONPATH so ``python -m evals.proxy`` works from any cwd. + + ``evals`` is not an installed package (pyproject excludes it); the MCP child + is often launched from a foreign temp dir (OpenCode project dir, etc.). + """ + root = str(REPO_ROOT) + out = dict(env) + existing = out.get("PYTHONPATH", "") + parts = [p for p in existing.split(os.pathsep) if p] + if root not in parts: + out["PYTHONPATH"] = root + (os.pathsep + existing if existing else "") + return out + + +def load_proxy_sidecar( + path: Path, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Load sidecar call rows (sorted by seq) plus a status dict. + + Status keys: + - missing / empty / complete / incomplete + - torn_line: final line failed to parse + - meta: proxy_meta row if present + - pending_left: from meta when present + """ + status: dict[str, Any] = { + "state": "missing", + "torn_line": False, + "meta": None, + "pending_left": None, + } + if not path.is_file(): + return [], status + try: + raw = path.read_bytes() + except OSError: + return [], status + if not raw: + status["state"] = "empty" + return [], status + + # Decode with replacement so invalid UTF-8 does not crash the loader. + text = raw.decode("utf-8", errors="replace") + lines = text.splitlines() + calls: list[dict[str, Any]] = [] + meta: dict[str, Any] | None = None + torn = False + for i, line in enumerate(lines): + s = line.strip() + if not s: + continue + try: + row = json.loads(s) + except json.JSONDecodeError: + # Tolerate a torn final line (crash mid-write); stop there. + if i == len(lines) - 1: + torn = True + break + continue + if not isinstance(row, dict): + continue + if row.get("row_type") == "proxy_meta": + meta = row + continue + tool = row.get("tool") + if not tool: + continue + calls.append( + { + "tool": str(tool), + "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), + "origin": "plane", + "is_error": bool(row.get("is_error")), + "result_chars": int(row.get("result_chars") or 0), + "duration_ms": row.get("duration_ms"), + "seq": row.get("seq"), + } + ) + + # Score order must match request seq, not response-append order. + calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) + + status["torn_line"] = torn + status["meta"] = meta + if meta is not None: + status["pending_left"] = meta.get("pending_left") + status["pumps_alive"] = bool(meta.get("pumps_alive")) + incomplete = bool( + torn + or meta is None + or (meta is not None and int(meta.get("pending_left") or 0) > 0) + or (meta is not None and bool(meta.get("pumps_alive"))) + ) + if not calls and not meta and not torn: + status["state"] = "empty" + elif incomplete: + status["state"] = "incomplete" + else: + status["state"] = "complete" + return calls, status + + +def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: + """Convenience: call rows only (sorted by seq).""" + calls, _status = load_proxy_sidecar(path) + return calls + + +def apply_proxy_sidecar( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: + """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. + + Incomplete sidecar (torn line, missing meta, pending_left>0) yields to the + CLI trace when the CLI has *more* plane calls. Returns + ``(plane_calls, client_calls, call_source)``. + """ + proxy_calls, status = load_proxy_sidecar(sidecar_path) + state = status.get("state") + if state in ("missing", "empty"): + notes.append("proxy_sidecar_empty") + return calls, client_calls, "json" + if state == "incomplete": + notes.append( + "proxy_sidecar_incomplete" + + (":torn" if status.get("torn_line") else "") + + (":no_meta" if status.get("meta") is None else "") + + (f":pending_left={status.get('pending_left')}" if status.get("pending_left") else "") + + (":pumps_alive" if status.get("pumps_alive") else "") + ) + if len(calls) > len(proxy_calls): + notes.append("proxy_sidecar_deferred_to_cli_trace") + return calls, client_calls, "json" + if proxy_calls: + notes.append(f"calls_from_proxy:{sidecar_path}") + return proxy_calls, client_calls, "proxy" + return calls, client_calls, "json" + # complete + notes.append(f"calls_from_proxy:{sidecar_path}") + return proxy_calls, client_calls, "proxy" + + +def wait_for_proxy_meta( + sidecar_path: Path, + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> bool: + """Poll until the sidecar gains a ``proxy_meta`` row (or the wait expires). + + After a CLI timeout the driver kills the CLI; the proxy is a *separate* + process that only then sees stdin EOF and needs up to + ``SHUTDOWN_DEADLINE_S`` to flush call rows + meta. Call this **before** + harvesting so the temp dir is not deleted mid-finalization. + + Returns True if meta was observed. + """ + # Local import keeps drivers import-light for non-proxy unit tests. + from evals.proxy import SHUTDOWN_DEADLINE_S + + if max_wait_s is None: + max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 + deadline = time.monotonic() + max_wait_s + while True: + _, status = load_proxy_sidecar(sidecar_path) + if status.get("meta") is not None: + return True + rem = deadline - time.monotonic() + if rem <= 0: + break + time.sleep(min(poll_s, rem)) + _, status = load_proxy_sidecar(sidecar_path) + return status.get("meta") is not None + + +def harvest_proxy_after_cli_timeout( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], + *, + max_wait_s: float | None = None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: + """Wait for proxy finalization after CLI kill, then harvest the sidecar. + + If meta never appears within the wait window, harvest anyway (incomplete + note from ``apply_proxy_sidecar``). ``max_wait_s`` defaults to + ``SHUTDOWN_DEADLINE_S + 2`` (see ``wait_for_proxy_meta``). + """ + found = wait_for_proxy_meta(sidecar_path, max_wait_s=max_wait_s) + if not found: + notes.append("proxy_meta_wait_timeout") + return apply_proxy_sidecar(calls, client_calls, sidecar_path, notes) + + +__all__ = [ + "apply_proxy_sidecar", + "ensure_proxy_pythonpath", + "harvest_proxy_after_cli_timeout", + "load_proxy_sidecar", + "load_proxy_sidecar_calls", + "proxy_wrap_server_command", + "wait_for_proxy_meta", +] diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index 52071947..2b90cc92 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -402,7 +402,7 @@ def test_find_codex_rollout_exact_match_and_unmatched(tmp_path: Path, monkeypatc encoding="utf-8", ) - monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) found = drivers_mod.find_codex_rollout(tid) assert found is not None assert tid in found.name @@ -423,7 +423,7 @@ def test_find_codex_rollout_session_meta_id(tmp_path: Path, monkeypatch): json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", encoding="utf-8", ) - monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) found = drivers_mod.find_codex_rollout("sess-meta-42") assert found is not None assert found.name == "rollout-meta-only.jsonl" @@ -431,11 +431,10 @@ def test_find_codex_rollout_session_meta_id(tmp_path: Path, monkeypatch): def test_codex_driver_notes_rollout_unmatched_when_no_file(tmp_path: Path, monkeypatch): """When thread_id is known but no rollout file matches, note codex_rollout_unmatched.""" - from evals import drivers as drivers_mod # Empty sessions dir under fake home (tmp_path / ".codex" / "sessions").mkdir(parents=True) - monkeypatch.setattr(drivers_mod.Path, "home", lambda: tmp_path) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") @@ -729,6 +728,38 @@ def test_agent_run_hit_max_maps_to_hit_max_iterations(): assert out["stop_reason"] == "max_turns" +def test_agent_run_to_harness_dict_does_not_guess_usage_total(): + """Generic row mapping must not invent usage_total from a vendor usage dict. + + Drivers own normalization (ClaudeCliDriver via normalize_claude_usage, + CodexCliDriver builds its own). Missing usage_total stays None. + """ + run = AgentRun( + calls=[], + final_text="ok", + usage={ + "input_tokens": 5000, + "output_tokens": 200, + # Codex-ish shape — not Claude modelUsage. A Claude rebuild would + # silently produce a wrong / empty total if reintroduced. + "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, + }, + usage_total=None, + stopped_reason="completed", + usage_scope="run", + call_source="stream", + ) + out = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=classify_call, + skip_result_tokens=True, + ) + assert out["usage"] == run.usage + assert out["usage_total"] is None + + # --------------------------------------------------------------------------- # Plumbing # --------------------------------------------------------------------------- @@ -832,7 +863,7 @@ def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): import signal from types import SimpleNamespace - from evals.drivers import _kill_process_group + from evals.drivers import kill_process_group pidfile = tmp_path / "pids.txt" script = tmp_path / "sticky_leader.py" @@ -880,7 +911,7 @@ def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): t0 = time.monotonic() # Direct killpg(leader_pid) — pgid == original leader pid under start_new_session. - ok = _kill_process_group(SimpleNamespace(pid=leader_pid)) + ok = kill_process_group(SimpleNamespace(pid=leader_pid)) assert ok is True assert time.monotonic() - t0 < 3.0 From 6af196cfde2ea8f39a570838dc6059900eb54d62 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 01:28:45 +0530 Subject: [PATCH 03/93] Replace the Anthropic-only SDK path with a generic API driver The in-process "sdk" path was not a driver: it lived inline in run.py, and get_driver returned None for it, so run_live carried two parallel code paths that had to be kept in sync. It also delegated the entire agent loop to a beta API (client.beta.messages.tool_runner plus the SDK's MCP tool conversion), which made one vendor's message shapes structural to the harness. The loop is now ours and provider-neutral. A backend owns conversation state and wire format behind start/next_turn/add_tool_results, so the driver deals in ToolCall and ToolResult and never sees a provider's block types. Anthropic runs on the stable Messages API; an OpenAI backend ships alongside it, importing lazily and taking an injectable client so the package stays uninstalled and the tests stay offline. Adding a provider is now one module. Because the driver executes tools itself it holds every result in full, so response-token cost no longer needs a provider endpoint: a backend may supply a counter, otherwise the count is estimated from the text and the row says so rather than quietly reporting an estimate as measured. The behaviour that carries correctness is preserved and now tested directly: tools are never executed on a refusal-terminated turn, results pair to calls by id rather than ordinal with a mismatch flag when ids do not line up, and iteration exhaustion is only flagged mid-tool-loop. Every row field report.py and existing result files depend on is unchanged. Co-Authored-By: Claude Fable 5 --- evals/DESIGN.md | 121 +++----- evals/README.md | 17 +- evals/drivers/__init__.py | 18 +- evals/drivers/api/__init__.py | 17 ++ evals/drivers/api/anthropic.py | 127 ++++++++ evals/drivers/api/backend.py | 65 ++++ evals/drivers/api/driver.py | 354 ++++++++++++++++++++++ evals/drivers/api/openai.py | 159 ++++++++++ evals/drivers/base.py | 52 +++- evals/report.py | 4 +- evals/run.py | 399 ++++++------------------- pyproject.toml | 4 +- tests/test_evals_api_driver.py | 531 +++++++++++++++++++++++++++++++++ tests/test_evals_drivers.py | 11 +- tests/test_evals_hardening.py | 1 + tests/test_evals_proxy.py | 2 + tests/test_evals_surface.py | 10 - 17 files changed, 1457 insertions(+), 435 deletions(-) create mode 100644 evals/drivers/api/__init__.py create mode 100644 evals/drivers/api/anthropic.py create mode 100644 evals/drivers/api/backend.py create mode 100644 evals/drivers/api/driver.py create mode 100644 evals/drivers/api/openai.py create mode 100644 tests/test_evals_api_driver.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 9ac30359..dce95fe2 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -24,68 +24,31 @@ The harness must support A/B comparison: same tasks against different tool surfa ## Architecture -**Driver:** the Anthropic Python SDK's beta tool runner with its MCP conversion helpers — -NOT the Claude Agent SDK, NOT a hand-rolled agent loop. This measures the MCP surface in -isolation (no coding-harness system prompt or built-in tools polluting the numbers). +**Driver:** `ApiDriver` owns the model/tool loop and the stdio MCP session. Provider adapters +own only conversation state and wire translation behind a neutral `ModelBackend` protocol: +`start(system, prompt, tools)`, `next_turn()`, and `add_tool_results(results)`. Anthropic uses +the stable Messages API; OpenAI uses Chat Completions function tools. CLI drivers keep the +same `AgentDriver` boundary, so `run.py` has one execution path for every driver. -The exact pattern (this is documented SDK API — do not improvise alternatives): - -```python -from anthropic import AsyncAnthropic -from anthropic.lib.tools.mcp import async_mcp_tool -from mcp import ClientSession -from mcp.client.stdio import stdio_client, StdioServerParameters - -client = AsyncAnthropic() # resolves ANTHROPIC_API_KEY / ant-auth profile from env - -server_params = StdioServerParameters( - command=sys.executable, - args=["-m", "plane_mcp", "stdio"], - env={ - "PLANE_API_KEY": os.environ["EVAL_PLANE_API_KEY"], - "PLANE_WORKSPACE_SLUG": os.environ["EVAL_PLANE_WORKSPACE_SLUG"], - "PLANE_BASE_URL": os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so"), - }, -) - -async with stdio_client(server_params) as (read, write): - async with ClientSession(read, write) as mcp_client: - await mcp_client.initialize() - tools_result = await mcp_client.list_tools() - runner = client.beta.messages.tool_runner( # sync call — returns the runner - model=MODEL_ID, - max_tokens=8192, - max_iterations=15, # turn cap per task - system=SYSTEM_PREAMBLE, - messages=[{"role": "user", "content": task["prompt"]}], - tools=[async_mcp_tool(t, mcp_client) for t in tools_result.tools], - ) - async for message in runner: - # capture tool_use blocks from message.content - for block in message.content: - if block.type == "tool_use": - record_call(block.name, block.input) - # capture tool results (cached — tools still run exactly once) - tool_response = runner.generate_tool_call_response() - if tool_response is not None: - record_results(tool_response) # user message w/ tool_result blocks - final = message -``` +Neutral turns contain final text, tool calls keyed by provider call ID, normalized usage, +and stop reason. The driver records calls, executes MCP tools, pairs results back by call ID, +and passes neutral results to the backend for the next provider turn. This isolates tool- +surface behavior without a coding-harness system prompt or built-in tools. Notes: - One fresh stdio server subprocess per task run (cheap, isolates state). -- Record `message.usage` from **every** yielded message (input_tokens, output_tokens, +- Record provider usage from **every** model turn (input tokens, output tokens, cache_read_input_tokens, cache_creation_input_tokens) — this is the exact context cost, returned free; the per-result counts below are a size proxy, not the cost figure. - Record the final message's `stop_reason`, and whether the loop ended by exhausting `max_iterations` — a capped/truncated run must be distinguishable from a genuine failure. - Detect the cap from `stop_reason` (the runner can legitimately finish with `end_turn` on + Detect the cap from `stop_reason` (a model can legitimately finish with `end_turn` on exactly its last permitted iteration — an unconditional iteration-count check misreports that as capped). -- **Never call `generate_tool_call_response()` on a refusal-terminated message** - (`stop_reason == "refusal"`): the SDK deliberately skips executing those tool_use blocks - (side effects the model never confirmed), and calling it from the loop body bypasses that - guard and fires real writes at the eval workspace. +- **Never execute tools on a refusal-terminated turn** (`stop_reason == "refusal"`), even if + the response also contains tool calls. Record the calls for auditability, then stop. +- Pair every tool result to its call by call ID, never list position. Missing, duplicate, or + unknown IDs set `result_pair_mismatch`. - `wall_time_s` measures the agent loop only: start the clock after `list_tools()` returns, stop it when the loop exits — MCP subprocess spawn/teardown and post-loop token counting are excluded. @@ -93,48 +56,44 @@ Notes: `PLANE_*` vars) — never inherit `os.environ`. `plane_mcp/client.py` prefers `PLANE_INTERNAL_BASE_URL` over `PLANE_BASE_URL`, so an inherited value silently points the agent at a different Plane instance than seed/verify. -- A harness/API failure (SDK exception, MCP crash) is recorded as `error: ""` on the +- A harness/API failure (provider exception, MCP crash) is recorded as `error: ""` on the row — it is neither a task failure nor a skip, and the row's zeroed metrics must not enter any statistic. - `SYSTEM_PREAMBLE` names the eval workspace slug and project name, states "complete the task using the available tools, then stop", and nothing else. Keep it under 100 words — it is part of the measured context. -- Omit `thinking` and sampling params entirely (adaptive thinking is the default on - claude-sonnet-5; `temperature`/`top_p`/`top_k` are rejected). -- Final assistant text = the last yielded message's text blocks (used by read-task verifiers). +- Omit thinking and sampling parameters; the API backend only sets the model, token cap, + system/instructions, conversation, and tools. +- Final assistant text = the last model turn's text (used by read-task verifiers). -**Token counting of tool results:** use the API's count_tokens endpoint, never tiktoken -(wrong tokenizer for Claude, ~15-20% off): - -```python -n = ( - await client.messages.count_tokens( - model=MODEL_ID, - messages=[{"role": "user", "content": result_text}], - ) -).input_tokens -``` +**Token sizing of tool results:** the owned loop holds the complete text passed back to the +model, so `result_chars` is always exact. A backend may expose a token counter; otherwise the +driver uses a deterministic character estimate and sets `result_tokens_estimated: true` on +the row. No provider token-count endpoint is required per tool result. Rules: - Run these counts **after** the agent loop finishes, not inline — they must not pollute `wall_time_s`. Buffer the raw result strings during the run, count at the end. -- A tool_result's content may be a list of blocks. Concatenate the text of `text` blocks; - for non-text blocks (e.g. image) record `result_kind: "image"` with `result_tokens: null` - and `result_chars` of the raw payload. `is_error` results are counted like text. +- A tool result's content may be a list of blocks. Text-only blocks are concatenated; + non-text or mixed content is serialized into the exact string sent to the model and marked + `result_kind: "image"` or `"mixed"`. Error results are sized like successful results. - Also record raw `len(chars)` alongside every count. -**Models** (CLI aliases → IDs; these are deliberate, do not substitute): +**API model aliases** (provider-specific): -| alias | model id | role | -|---|---|---| -| `sonnet` | `claude-sonnet-5` | default / representative agent | -| `haiku` | `claude-haiku-4-5` | canary — weaker models amplify tool-surface defects | +| provider | alias | model id | role | +|---|---|---|---| +| Anthropic | `sonnet` | `claude-sonnet-5` | default / representative agent | +| Anthropic | `haiku` | `claude-haiku-4-5` | weaker-model canary | +| OpenAI | `sonnet` | `gpt-5` | representative agent | +| OpenAI | `haiku` | `gpt-5-mini` | faster/weaker-model canary | ## Environment | var | purpose | |---|---| -| `ANTHROPIC_API_KEY` | driver LLM auth (or an `ant auth` profile) | +| `ANTHROPIC_API_KEY` | default API-provider authentication | +| `OPENAI_API_KEY` | OpenAI provider authentication when its optional SDK is installed | | `EVAL_PLANE_API_KEY` | Plane API key for the **dedicated eval workspace** | | `EVAL_PLANE_WORKSPACE_SLUG` | eval workspace slug — never a production workspace | | `EVAL_PLANE_BASE_URL` | optional, defaults to `https://api.plane.so` | @@ -149,6 +108,7 @@ Construct the client the same way `plane_mcp/client.py` does for stdio mode, but evals/ __init__.py DESIGN.md # this file + drivers/api/ # owned loop + neutral, Anthropic, and OpenAI backends tasks.py # task definitions (plain dicts) + verifier functions seed.py # per-run fixture create/teardown via plane-sdk run.py # CLI driver (python -m evals.run) @@ -160,13 +120,12 @@ Dependencies: add to `pyproject.toml`: ```toml [project.optional-dependencies] -evals = ["anthropic[mcp]>="] +evals = ["anthropic>=0.121.0"] ``` -Pin a `>=` floor at whatever the current anthropic release is when you implement, and -verify `from anthropic.lib.tools.mcp import async_mcp_tool` actually imports in the venv. -Do NOT change the existing `mcp==1.26.0` pin — `anthropic[mcp]` must coexist with it. -No other new dependencies. Stdlib only otherwise (argparse, json, asyncio, uuid, time). +Pin a `>=` floor for the stable Anthropic Messages client. OpenAI support stays optional: +the module imports its SDK lazily only when that provider is selected, and tests inject a +fake client. Do NOT change the existing `mcp==1.26.0` pin. ## Task schema (`tasks.py`) diff --git a/evals/README.md b/evals/README.md index 0dc4494c..df1298a3 100644 --- a/evals/README.md +++ b/evals/README.md @@ -32,11 +32,17 @@ they were the optimal ones), errors, and the agent's final text. unset REDIS_HOST REDIS_PORT # else the SDK client picks up a stale cache config ``` -3. **An agent CLI** for the driver you pick (below), already authenticated. +3. **Model access for the driver you pick.** The API driver uses + `ANTHROPIC_API_KEY` by default; OpenAI requires its SDK and `OPENAI_API_KEY`. + CLI drivers require their corresponding local CLI to already be authenticated. ## Running ```bash +# Provider-neutral API loop (default provider: Anthropic) +.venv/bin/python -m evals.run --driver api --provider anthropic --model sonnet \ + --surface full --out results/api.jsonl + # Everything, one surface .venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ --surface full --out results/legacy.jsonl @@ -70,15 +76,22 @@ compared, and it is honest about what it ran because it launches the server you | Driver | Backend | Notes | |---|---|---| +| `api` | Owned API + MCP loop | Provider-neutral; `--provider anthropic` (default) or `openai` | +| `sdk` | Alias for `api` | Retained for old commands/result pipelines | | `codex-cli` | OpenAI Codex CLI | Pass a real model id (`gpt-5.6-sol`); the short-alias table is incomplete | | `claude-cli` | Claude Code CLI | `--model sonnet` / `haiku` | | `antigravity-cli` | Antigravity CLI (`agy`) | Runs under a synthetic HOME so its MCP config is ours, not yours | | `opencode-cli` | opencode | Temp project config per run | -| `sdk` | Anthropic SDK tool runner | No coding-harness prompt in the way; needs `ANTHROPIC_API_KEY` | Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool calls are counted from the wire rather than from whatever the agent claims it did. +The API driver executes MCP calls itself, records exact result character counts, and sizes +result tokens without making a provider request per result. A backend may supply a token +counter; otherwise rows set `result_tokens_estimated: true` and use a deterministic +character-based estimate. CLI drivers retain null result-token counts because they do not +always expose complete tool result text. + ### Reading results ```bash diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 00c56d75..8b338755 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -1,8 +1,8 @@ """Agent-driver abstraction for the Plane MCP eval harness. Drivers run one task against a tool surface and return a normalized -``AgentRun``. The default ``sdk`` driver preserves the historical Anthropic -SDK + in-process MCP client path. CLI drivers (``claude-cli``, ``codex-cli``, +``AgentRun``. The default ``api`` driver owns a provider-neutral model/tool +loop over an in-process MCP client. CLI drivers (``claude-cli``, ``codex-cli``, ``antigravity-cli``, ``opencode-cli``) spawn locally installed agent CLIs on the user's subscription — no Anthropic API key required for those paths. @@ -43,6 +43,7 @@ prepare_antigravity_fake_home, write_antigravity_mcp_config, ) +from evals.drivers.api import ApiDriver from evals.drivers.base import ( REPO_ROOT, AgentDriver, @@ -83,14 +84,14 @@ # Registry # --------------------------------------------------------------------------- -KNOWN_DRIVERS = frozenset({"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) +KNOWN_DRIVERS = frozenset({"api", "sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) -def get_driver(name: str, **kwargs: Any) -> AgentDriver | None: - """Return a driver instance, or None for the in-process ``sdk`` path.""" - key = (name or "sdk").strip().lower() - if key == "sdk": - return None # handled inline in evals.run +def get_driver(name: str, **kwargs: Any) -> AgentDriver: + """Return a driver instance; ``sdk`` is a legacy alias for ``api``.""" + key = (name or "api").strip().lower() + if key in ("api", "sdk"): + return ApiDriver(**kwargs) if key == "claude-cli": return ClaudeCliDriver(**kwargs) if key == "codex-cli": @@ -107,6 +108,7 @@ def get_driver(name: str, **kwargs: Any) -> AgentDriver | None: "AgentDriver", "AgentRun", "AntigravityCliDriver", + "ApiDriver", "ClaudeCliDriver", "CodexCliDriver", "OpencodeCliDriver", diff --git a/evals/drivers/api/__init__.py b/evals/drivers/api/__init__.py new file mode 100644 index 00000000..a3e88491 --- /dev/null +++ b/evals/drivers/api/__init__.py @@ -0,0 +1,17 @@ +"""Provider-generic API driver and backend translations.""" + +from evals.drivers.api.anthropic import AnthropicBackend +from evals.drivers.api.backend import ModelBackend, ToolCall, ToolResult, ToolSpec, Turn +from evals.drivers.api.driver import ApiDriver +from evals.drivers.api.openai import OpenAIBackend + +__all__ = [ + "AnthropicBackend", + "ApiDriver", + "ModelBackend", + "OpenAIBackend", + "ToolCall", + "ToolResult", + "ToolSpec", + "Turn", +] diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py new file mode 100644 index 00000000..aa6143b6 --- /dev/null +++ b/evals/drivers/api/anthropic.py @@ -0,0 +1,127 @@ +"""Anthropic Messages API translation for the provider-neutral eval loop.""" + +from __future__ import annotations + +from typing import Any + +from evals.drivers.api.backend import ToolCall, ToolResult, ToolSpec, Turn + + +def _field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def _usage_dict(usage: Any) -> dict[str, int] | None: + if usage is None: + return None + return { + "in": int(_field(usage, "input_tokens", 0) or 0), + "out": int(_field(usage, "output_tokens", 0) or 0), + "cache_read": int(_field(usage, "cache_read_input_tokens", 0) or 0), + "cache_write": int(_field(usage, "cache_creation_input_tokens", 0) or 0), + } + + +class AnthropicBackend: + """Stateful adapter over stable ``client.messages.create`` calls.""" + + provider = "anthropic" + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + if client is None: + from anthropic import Anthropic + + client = Anthropic() + self.client = client + self.model = model + self.actual_model = model + self.max_tokens = max_tokens + self.system: str | None = None + self.messages: list[dict[str, Any]] = [] + self.tools: list[dict[str, Any]] = [] + self.started = False + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.system = system + self.messages = [{"role": "user", "content": prompt}] + self.tools = [ + { + "name": tool.name, + "description": tool.description, + "input_schema": tool.input_schema, + } + for tool in tools + ] + self.started = True + + def next_turn(self) -> Turn: + if not self.started: + raise RuntimeError("AnthropicBackend.start() must be called before next_turn()") + request: dict[str, Any] = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": self.messages, + "tools": self.tools, + } + if self.system is not None: + request["system"] = self.system + message = self.client.messages.create(**request) + content = _field(message, "content", None) or [] + # Replay the provider's content objects verbatim, including thinking or + # other blocks required by later Messages API turns. + self.messages.append({"role": "assistant", "content": content}) + + text_parts: list[str] = [] + calls: list[ToolCall] = [] + for block in content: + block_type = _field(block, "type") + if block_type == "text": + text = _field(block, "text") + if text: + text_parts.append(str(text)) + elif block_type == "refusal": + explanation = _field(block, "explanation") + if explanation: + text_parts.append(str(explanation)) + elif block_type == "tool_use": + args = _field(block, "input", {}) or {} + if not isinstance(args, dict): + args = {"_raw": args} + calls.append( + ToolCall( + id=str(_field(block, "id", "") or ""), + name=str(_field(block, "name", "") or ""), + args=args, + ) + ) + + response_model = _field(message, "model") + if response_model: + self.actual_model = str(response_model) + return Turn( + text="\n".join(text_parts), + tool_calls=calls, + usage=_usage_dict(_field(message, "usage")), + stop_reason=_field(message, "stop_reason"), + ) + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.messages.append( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": result.call_id, + "content": result.text, + "is_error": result.is_error, + } + for result in results + ], + } + ) + + +__all__ = ["AnthropicBackend"] diff --git a/evals/drivers/api/backend.py b/evals/drivers/api/backend.py new file mode 100644 index 00000000..ebd2001a --- /dev/null +++ b/evals/drivers/api/backend.py @@ -0,0 +1,65 @@ +"""Provider-neutral types for API-backed eval agent loops.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class ToolSpec: + """A model-facing tool definition translated from MCP ``list_tools``.""" + + name: str + description: str + input_schema: dict[str, Any] + + +@dataclass(frozen=True) +class ToolCall: + """A provider-neutral model request to invoke one MCP tool.""" + + id: str + name: str + args: dict[str, Any] + + +@dataclass(frozen=True) +class ToolResult: + """A provider-neutral MCP result paired to its model call ID.""" + + call_id: str + text: str + is_error: bool = False + kind: str = "text" + + +@dataclass(frozen=True) +class Turn: + """One normalized assistant response from a model provider.""" + + text: str + tool_calls: list[ToolCall] + usage: dict[str, int] | None + stop_reason: str | None + + +class ModelBackend(Protocol): + """Conversation-owning adapter for one model provider. + + Backends retain all provider wire state. The driver sees only normalized + turns and adds normalized tool results after executing MCP calls. + """ + + provider: str + model: str + actual_model: str + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: ... + + def next_turn(self) -> Turn: ... + + def add_tool_results(self, results: list[ToolResult]) -> None: ... + + +__all__ = ["ModelBackend", "ToolCall", "ToolResult", "ToolSpec", "Turn"] diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py new file mode 100644 index 00000000..236dd3b4 --- /dev/null +++ b/evals/drivers/api/driver.py @@ -0,0 +1,354 @@ +"""Owned provider-neutral agent loop over a stdio MCP session.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import sys +import time +from collections.abc import Callable +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +from mcp import ClientSession +from mcp.client.stdio import StdioServerParameters, stdio_client + +from evals.drivers.api.anthropic import AnthropicBackend +from evals.drivers.api.backend import ModelBackend, ToolResult, ToolSpec +from evals.drivers.api.openai import OpenAIBackend +from evals.drivers.base import AgentRun + +DEFAULT_MAX_TOKENS = 8192 +KNOWN_API_PROVIDERS = frozenset({"anthropic", "openai"}) + +BackendFactory = Callable[[str, int], ModelBackend] +McpSessionFactory = Callable[[StdioServerParameters], Any] + + +def estimate_tokens(text: str) -> int: + """Estimate token count from recorded text without a provider request.""" + if not text: + return 0 + return max(1, (len(text) + 3) // 4) + + +def tool_spec_from_mcp(tool: Any) -> ToolSpec: + """Translate an MCP list-tools entry into a neutral tool specification.""" + if isinstance(tool, dict): + name = tool.get("name") or "" + description = tool.get("description") or "" + schema = tool.get("inputSchema") or tool.get("input_schema") or {"type": "object"} + else: + name = getattr(tool, "name", "") or "" + description = getattr(tool, "description", "") or "" + schema = getattr(tool, "inputSchema", None) or getattr(tool, "input_schema", None) + schema = schema or {"type": "object"} + if not isinstance(schema, dict): + schema = {"type": "object"} + return ToolSpec(name=str(name), description=str(description), input_schema=schema) + + +def _dump_content_block(block: Any) -> Any: + if isinstance(block, (dict, str, int, float, bool)) or block is None: + return block + dump = getattr(block, "model_dump", None) + if callable(dump): + return dump(by_alias=True, exclude_none=True) + return str(block) + + +def _content_text_and_kind(content: Any) -> tuple[str, str]: + if content is None: + return "", "text" + if isinstance(content, str): + return content, "text" + if not isinstance(content, list): + return str(content), "text" + + text_parts: list[str] = [] + saw_non_text = False + for block in content: + block_type = block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + if block_type == "text" or block_type is None: + text = block.get("text") if isinstance(block, dict) else getattr(block, "text", None) + if text is None and isinstance(block, str): + text = block + if text is not None: + text_parts.append(str(text)) + else: + saw_non_text = True + + if not saw_non_text: + return "\n".join(text_parts), "text" + payload = json.dumps([_dump_content_block(block) for block in content], default=str, separators=(",", ":")) + return payload, "mixed" if text_parts else "image" + + +def tool_result_from_mcp(call_id: str, raw_result: Any) -> ToolResult: + """Translate an MCP call result, preserving an injected result ID for tests.""" + if isinstance(raw_result, ToolResult): + return raw_result + if isinstance(raw_result, dict): + content = raw_result.get("content") + is_error = bool(raw_result.get("isError") or raw_result.get("is_error")) + else: + content = getattr(raw_result, "content", None) + is_error = bool(getattr(raw_result, "isError", False) or getattr(raw_result, "is_error", False)) + text, kind = _content_text_and_kind(content) + return ToolResult(call_id=call_id, text=text, is_error=is_error, kind=kind) + + +class ApiDriver: + """Run an owned tool loop against Anthropic or OpenAI and a stdio MCP server.""" + + name = "api" + + def __init__( + self, + *, + provider: str = "anthropic", + client: Any | None = None, + backend_factory: BackendFactory | None = None, + mcp_session_factory: McpSessionFactory | None = None, + server_command: list[str] | None = None, + python_bin: str | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + ) -> None: + provider = provider.strip().lower() + if provider not in KNOWN_API_PROVIDERS: + raise ValueError(f"unknown API provider {provider!r}; expected one of {sorted(KNOWN_API_PROVIDERS)}") + if server_command is not None and not server_command: + raise ValueError("server_command cannot be empty") + self.provider = provider + self.client = client + self.backend_factory = backend_factory + self.mcp_session_factory = mcp_session_factory + self.server_command = list(server_command) if server_command is not None else None + self.python_bin = python_bin or sys.executable + self.max_tokens = max_tokens + + def _make_backend(self, model: str) -> ModelBackend: + if self.backend_factory is not None: + return self.backend_factory(model, self.max_tokens) + if self.provider == "anthropic": + backend = AnthropicBackend(model, max_tokens=self.max_tokens, client=self.client) + else: + backend = OpenAIBackend(model, max_tokens=self.max_tokens, client=self.client) + # Delay credential-dependent client creation until the first non-skipped + # task, then reuse the provider's connection pool across the battery. + if self.client is None: + self.client = backend.client + return backend + + def _server_params(self, mcp_env: dict[str, str], cwd: Path | None) -> StdioServerParameters: + command = self.server_command or [self.python_bin, "-m", "plane_mcp", "stdio"] + return StdioServerParameters( + command=command[0], + args=command[1:], + env=mcp_env, + cwd=cwd, + ) + + @asynccontextmanager + async def _mcp_session(self, params: StdioServerParameters): + if self.mcp_session_factory is not None: + context = self.mcp_session_factory(params) + if inspect.isawaitable(context): + context = await context + if hasattr(context, "__aenter__"): + async with context as session: + yield session + else: + yield context + return + + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + yield session + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + if not model: + raise ValueError("the API driver requires a model ID") + if max_turns < 1: + raise ValueError("max_turns must be at least 1") + return asyncio.run( + self._run_task( + prompt=prompt, + mcp_env=mcp_env, + model=model, + max_turns=max_turns, + system=system, + cwd=cwd, + ) + ) + + async def _run_task( + self, + *, + prompt: str, + mcp_env: dict[str, str], + model: str, + max_turns: int, + system: str | None, + cwd: Path | None, + ) -> AgentRun: + backend = self._make_backend(model) + calls: list[dict[str, Any]] = [] + pending_results: list[tuple[int, str]] = [] + usage_per_iteration: list[dict[str, int]] = [] + result_pair_mismatch = False + hit_max_iterations = False + iterations = 0 + final_text = "" + stop_reason: str | None = None + + params = self._server_params(mcp_env, cwd) + async with self._mcp_session(params) as mcp_client: + await mcp_client.initialize() + tools_result = await mcp_client.list_tools() + raw_tools = ( + tools_result.get("tools", []) if isinstance(tools_result, dict) else getattr(tools_result, "tools", []) + ) + backend.start(system, prompt, [tool_spec_from_mcp(tool) for tool in raw_tools]) + + # Match the historical metric: model/tool loop only, after list_tools. + started_at = time.perf_counter() + try: + while iterations < max_turns: + turn = backend.next_turn() + iterations += 1 + final_text = turn.text + stop_reason = turn.stop_reason + if turn.usage is not None: + usage_per_iteration.append(dict(turn.usage)) + + call_indices: dict[str, int] = {} + for tool_call in turn.tool_calls: + idx = len(calls) + calls.append( + { + "tool": tool_call.name, + "args": tool_call.args, + "result_tokens": None, + "result_chars": 0, + "result_kind": "text", + "is_error": False, + } + ) + if not tool_call.id or tool_call.id in call_indices: + result_pair_mismatch = True + else: + call_indices[tool_call.id] = idx + + # Record the model's calls, but never execute side effects on + # a refusal-terminated response. + if stop_reason == "refusal": + break + + if not turn.tool_calls: + if stop_reason == "pause_turn" and iterations < max_turns: + continue + break + + executed: list[tuple[ToolResult, float]] = [] + for tool_call in turn.tool_calls: + call_started = time.perf_counter() + raw_result = await mcp_client.call_tool(tool_call.name, arguments=tool_call.args) + duration_ms = round((time.perf_counter() - call_started) * 1000, 3) + executed.append((tool_result_from_mcp(tool_call.id, raw_result), duration_ms)) + + matched_ids: set[str] = set() + tool_results: list[ToolResult] = [] + for result, duration_ms in executed: + tool_results.append(result) + idx = call_indices.get(result.call_id) + if idx is None or result.call_id in matched_ids: + result_pair_mismatch = True + continue + matched_ids.add(result.call_id) + calls[idx]["result_chars"] = len(result.text) + calls[idx]["result_kind"] = result.kind + calls[idx]["is_error"] = result.is_error + calls[idx]["duration_ms"] = duration_ms + pending_results.append((idx, result.text)) + if matched_ids != set(call_indices) or len(call_indices) != len(turn.tool_calls): + result_pair_mismatch = True + + backend.add_tool_results(tool_results) + if iterations >= max_turns: + hit_max_iterations = stop_reason not in ("end_turn", "max_tokens") + break + if stop_reason != "tool_use": + break + finally: + wall_time_s = time.perf_counter() - started_at + + # Token sizing is intentionally outside wall_time. A backend may offer + # a local/exact counter; absence or failure falls back to recorded text. + counter = getattr(backend, "count_tokens", None) + token_count_failures = 0 + result_tokens_estimated = False + for idx, result_text in pending_results: + counted: int | None = None + if callable(counter): + try: + raw_count = counter(result_text) + if inspect.isawaitable(raw_count): + raw_count = await raw_count + counted = int(raw_count) + except Exception: + token_count_failures += 1 + if counted is None: + counted = estimate_tokens(result_text) + result_tokens_estimated = True + calls[idx]["result_tokens"] = counted + + usage_total = { + "input_tokens": sum(item.get("in", 0) for item in usage_per_iteration), + "output_tokens": sum(item.get("out", 0) for item in usage_per_iteration), + "cache_read_input_tokens": sum(item.get("cache_read", 0) for item in usage_per_iteration), + "cache_creation_input_tokens": sum(item.get("cache_write", 0) for item in usage_per_iteration), + "source": "iterations", + } + return AgentRun( + calls=calls, + final_text=final_text, + usage=usage_per_iteration[-1] if usage_per_iteration else None, + usage_total=usage_total, + usage_scope="iteration", + stopped_reason=stop_reason or "end_turn", + call_source="api", + hit_max_turns=hit_max_iterations, + wall_time_s=round(wall_time_s, 3), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=sum(item.get("in", 0) for item in usage_per_iteration), + result_pair_mismatch=result_pair_mismatch, + token_count_failures=token_count_failures, + result_tokens_estimated=result_tokens_estimated, + provider=str(getattr(backend, "provider", self.provider)), + model=str(getattr(backend, "actual_model", getattr(backend, "model", model))), + requested_model=model, + ) + + +__all__ = [ + "DEFAULT_MAX_TOKENS", + "KNOWN_API_PROVIDERS", + "ApiDriver", + "BackendFactory", + "McpSessionFactory", + "estimate_tokens", + "tool_result_from_mcp", + "tool_spec_from_mcp", +] diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py new file mode 100644 index 00000000..ae389fad --- /dev/null +++ b/evals/drivers/api/openai.py @@ -0,0 +1,159 @@ +"""OpenAI Chat Completions translation for the provider-neutral eval loop.""" + +from __future__ import annotations + +import json +from typing import Any + +from evals.drivers.api.backend import ToolCall, ToolResult, ToolSpec, Turn + + +def _field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def _usage_dict(usage: Any) -> dict[str, int] | None: + if usage is None: + return None + prompt_details = _field(usage, "prompt_tokens_details") + return { + "in": int(_field(usage, "prompt_tokens", 0) or 0), + "out": int(_field(usage, "completion_tokens", 0) or 0), + "cache_read": int(_field(prompt_details, "cached_tokens", 0) or 0), + "cache_write": 0, + } + + +def _normalize_stop_reason(finish_reason: str | None, refusal: Any) -> str | None: + if refusal or finish_reason == "content_filter": + return "refusal" + return { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "function_call": "tool_use", + }.get(str(finish_reason), finish_reason) + + +class OpenAIBackend: + """Stateful adapter over ``client.chat.completions.create``. + + ``openai`` is deliberately imported only when no client was injected, so + importing this module and all offline tests work without that package. + """ + + provider = "openai" + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + if client is None: + try: + from openai import OpenAI + except ImportError as exc: + raise RuntimeError( + "the OpenAI API provider requires the optional 'openai' package; " + "install it in the runtime that launches evals" + ) from exc + + client = OpenAI() + self.client = client + self.model = model + self.actual_model = model + self.max_tokens = max_tokens + self.messages: list[dict[str, Any]] = [] + self.tools: list[dict[str, Any]] = [] + self.started = False + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.messages = [] + if system is not None: + self.messages.append({"role": "system", "content": system}) + self.messages.append({"role": "user", "content": prompt}) + self.tools = [ + { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": tool.input_schema, + }, + } + for tool in tools + ] + self.started = True + + def next_turn(self) -> Turn: + if not self.started: + raise RuntimeError("OpenAIBackend.start() must be called before next_turn()") + completion = self.client.chat.completions.create( + model=self.model, + max_completion_tokens=self.max_tokens, + messages=self.messages, + tools=self.tools, + ) + choices = _field(completion, "choices", None) or [] + if not choices: + raise RuntimeError("OpenAI Chat Completions returned no choices") + choice = choices[0] + message = _field(choice, "message") + raw_calls = _field(message, "tool_calls", None) or [] + + calls: list[ToolCall] = [] + wire_calls: list[dict[str, Any]] = [] + for raw_call in raw_calls: + function = _field(raw_call, "function") + raw_args = _field(function, "arguments", "{}") or "{}" + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + args = {"_raw": raw_args} + elif isinstance(raw_args, dict): + args = raw_args + raw_args = json.dumps(raw_args, separators=(",", ":")) + else: + args = {"_raw": raw_args} + raw_args = json.dumps(raw_args, default=str) + if not isinstance(args, dict): + args = {"_raw": args} + call_id = str(_field(raw_call, "id", "") or "") + name = str(_field(function, "name", "") or "") + calls.append(ToolCall(id=call_id, name=name, args=args)) + wire_calls.append( + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": raw_args}, + } + ) + + content = _field(message, "content") + refusal = _field(message, "refusal") + assistant_message: dict[str, Any] = {"role": "assistant", "content": content} + if wire_calls: + assistant_message["tool_calls"] = wire_calls + self.messages.append(assistant_message) + + response_model = _field(completion, "model") + if response_model: + self.actual_model = str(response_model) + return Turn( + text=str(content or refusal or ""), + tool_calls=calls, + usage=_usage_dict(_field(completion, "usage")), + stop_reason=_normalize_stop_reason(_field(choice, "finish_reason"), refusal), + ) + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.messages.extend( + { + "role": "tool", + "tool_call_id": result.call_id, + "content": result.text, + } + for result in results + ) + + +__all__ = ["OpenAIBackend"] diff --git a/evals/drivers/base.py b/evals/drivers/base.py index 3ff0bed5..6ece231b 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -32,13 +32,23 @@ class AgentRun: client_tool_calls: list[dict[str, Any]] = field(default_factory=list) # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens usage_total: dict[str, Any] | None = None - # Harness extras (optional; defaults keep SDK path simple) + # Harness extras (optional; defaults keep CLI paths simple) usage_scope: str = "run" # 'run' | 'iteration' - call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'sdk' + call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'api' hit_max_turns: bool = False wall_time_s: float = 0.0 experimental: bool = False notes: list[str] = field(default_factory=list) + usage_per_iteration: list[dict[str, int]] = field(default_factory=list) + cum_input_tokens: int | None = None + result_pair_mismatch: bool = False + token_count_failures: int = 0 + # None means result token sizing was skipped (CLI drivers). False means a + # backend counter was used; True means at least one result used estimation. + result_tokens_estimated: bool | None = None + provider: str | None = None + model: str | None = None + requested_model: str | None = None class AgentDriver(Protocol): @@ -117,7 +127,7 @@ def split_plane_and_client_calls( """Partition tagged calls into plane vs client lists. Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls - (SDK path) default to plane so existing harness behavior is unchanged. + (API path) default to plane so existing harness behavior is unchanged. """ plane: list[dict[str, Any]] = [] client: list[dict[str, Any]] = [] @@ -130,7 +140,7 @@ def split_plane_and_client_calls( elif raw.startswith("mcp__"): origin = "client" # other MCP server else: - origin = "plane" # bare name → assume plane (SDK) + origin = "plane" # bare name → assume plane (API) if origin == "client": client.append(c) else: @@ -172,9 +182,9 @@ def agent_run_to_harness_dict( "tool": tool, "class": classify(str(tool), optimal, alternate), "args_chars": args_chars, - "result_tokens": None, + "result_tokens": None if skip_result_tokens else c.get("result_tokens"), "result_chars": int(c["result_chars"]) if c.get("result_chars") is not None else 0, - "result_kind": "text", + "result_kind": str(c.get("result_kind") or "text"), "is_error": bool(c.get("is_error")), } if c.get("duration_ms") is not None: @@ -184,7 +194,7 @@ def agent_run_to_harness_dict( if isinstance(args, dict) and isinstance(args.get("action"), str): rec["action"] = args["action"] if skip_result_tokens: - rec["result_tokens_skipped"] = "no API key / CLI driver has no count_tokens" + rec["result_tokens_skipped"] = "CLI driver does not expose complete result text" calls.append(rec) client_tool_calls: list[dict[str, Any]] = [] @@ -218,7 +228,15 @@ def agent_run_to_harness_dict( is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" usage_total = run.usage_total - if is_cli and skip_result_tokens: + if run.usage_per_iteration: + usage_per_iteration = [dict(item) for item in run.usage_per_iteration] + cum_input = ( + run.cum_input_tokens + if run.cum_input_tokens is not None + else sum(item.get("in", 0) for item in usage_per_iteration) + ) + cum_reason = None + elif is_cli and skip_result_tokens: cum_input: int | None = None cum_reason: str | None = ( "CLI driver: Claude usage.input_tokens is uncached-only; " @@ -230,9 +248,9 @@ def agent_run_to_harness_dict( cum_reason = None usage_per_iteration = [] if run.usage and run.usage_scope == "iteration": - pass # SDK fills this separately + pass - return { + result = { "final_text": run.final_text, "calls": calls, "num_calls": len(calls), @@ -250,18 +268,26 @@ def agent_run_to_harness_dict( "wall_time_s": run.wall_time_s, "stop_reason": stop_reason, "hit_max_iterations": hit_max, - "result_pair_mismatch": False, - "token_count_failures": 0, + "result_pair_mismatch": run.result_pair_mismatch, + "token_count_failures": run.token_count_failures, + "result_tokens_estimated": run.result_tokens_estimated, "usage_scope": run.usage_scope, "call_source": run.call_source, "driver_raw_ref": run.raw_ref, "driver_notes": list(run.notes), "result_tokens_skipped_reason": ( - "CLI driver: count_tokens requires Anthropic API key; skipped" if skip_result_tokens else None + "CLI driver: complete tool result text is unavailable; skipped" if skip_result_tokens else None ), "usage": run.usage, "usage_total": usage_total, } + if run.provider is not None: + result["provider"] = run.provider + if run.model is not None: + result["model"] = run.model + if run.requested_model is not None: + result["requested_model"] = run.requested_model + return result __all__ = [ diff --git a/evals/report.py b/evals/report.py index cc3843d7..177724f6 100644 --- a/evals/report.py +++ b/evals/report.py @@ -90,10 +90,10 @@ def is_meta_row(row: dict[str, Any]) -> bool: def is_infra_error_row(row: dict[str, Any]) -> bool: - """True when a row failed for infrastructure reasons (seed/cli/sdk), not task verify. + """True when a row failed for infrastructure reasons (seed/cli/api), not task verify. Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, - ``infra_sdk``, …) is excluded from success-rate denominators. + ``infra_api``, ``infra_sdk``, …) is excluded from success-rate denominators. """ ec = row.get("error_class") return isinstance(ec, str) and ec.startswith("infra_") diff --git a/evals/run.py b/evals/run.py index 7c5da868..c6146421 100644 --- a/evals/run.py +++ b/evals/run.py @@ -14,7 +14,6 @@ import os import subprocess import sys -import time import uuid from datetime import datetime, timezone from pathlib import Path @@ -41,6 +40,12 @@ "sonnet": "claude-sonnet-5", "haiku": "claude-haiku-4-5", } +API_MODEL_ALIASES: dict[str, dict[str, str]] = { + "anthropic": MODEL_ALIASES, + # Preserve the harness's representative/fast intent when the user switches + # providers without also overriding the historical sonnet/haiku aliases. + "openai": {"sonnet": "gpt-5", "haiku": "gpt-5-mini"}, +} # Per-driver resolution of the short harness aliases (sonnet/haiku). # Drivers that need provider/model form get qualified defaults; unknown # strings (e.g. ``anthropic/claude-…``) pass through unchanged. @@ -58,15 +63,16 @@ } -def resolve_model_for_driver(driver_name: str, model: str) -> str: +def resolve_model_for_driver(driver_name: str, model: str, *, provider: str = "anthropic") -> str: """Map a harness model token to the string the given driver expects. Known short aliases (sonnet/haiku) are looked up per-driver. Any other string (including already-qualified ``provider/model``) is passed through. """ - key = (driver_name or "sdk").strip().lower() - if key == "sdk": - return MODEL_ALIASES.get(model, model) + key = (driver_name or "api").strip().lower() + if key in ("api", "sdk"): + table = API_MODEL_ALIASES.get(provider.strip().lower()) or {} + return table.get(model, model) table = CLI_MODEL_ALIASES.get(key) or {} return table.get(model, model) @@ -113,64 +119,6 @@ def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: return "out_of_set" -def _tool_result_text(content: Any) -> tuple[str, str]: - """Return (text_for_counting, result_kind). - - result_kind is 'text' | 'image' | 'mixed'. Mixed keeps text for token counting - and records that non-text blocks were present (char length of full payload). - """ - if content is None: - return "", "text" - if isinstance(content, str): - return content, "text" - if isinstance(content, list): - texts: list[str] = [] - saw_non_text = False - for block in content: - btype = getattr(block, "type", None) or (block.get("type") if isinstance(block, dict) else None) - if btype == "text" or btype is None: - text = getattr(block, "text", None) or (block.get("text") if isinstance(block, dict) else None) - if text is None and isinstance(block, str): - text = block - if text is not None: - texts.append(str(text)) - else: - saw_non_text = True - joined = "\n".join(texts) - if texts and saw_non_text: - return joined, "mixed" - if texts: - return joined, "text" - if saw_non_text: - return json.dumps(content, default=str), "image" - return "", "text" - return str(content), "text" - - -async def _count_result_tokens(client: Any, model: str, result_text: str) -> int | None: - if not result_text: - return 0 - try: - n = await client.messages.count_tokens( - model=model, - messages=[{"role": "user", "content": result_text}], - ) - return n.input_tokens - except Exception as exc: - print(f"count_tokens warning: {exc}", file=sys.stderr) - return None - - -def _extract_final_text(message: Any) -> str: - if message is None: - return "" - parts: list[str] = [] - for block in getattr(message, "content", None) or []: - if getattr(block, "type", None) == "text" and getattr(block, "text", None): - parts.append(block.text) - return "\n".join(parts) - - def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: """Build MCP stdio env from scratch — never inherit os.environ (F6). @@ -224,8 +172,8 @@ def _resume_field_mismatch( raw = row.get(field) if raw is None or raw == "": return None # back-compat: older rows without the key pass - # surface/driver compare case-insensitively; battery/model are exact strings. - if field in ("surface", "driver"): + # Surface/driver/provider compare case-insensitively; battery/model are exact. + if field in ("surface", "driver", "provider"): got, want = str(raw).strip().lower(), expected.strip().lower() else: got, want = str(raw).strip(), expected.strip() @@ -275,12 +223,13 @@ def load_resume_skip_keys( battery: str | None = None, model: str | None = None, driver: str | None = None, + provider: str | None = None, ) -> tuple[set[tuple[str, int]], int, int]: """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / - battery / model / driver disagrees with the current run (missing keys pass for + battery / model / driver / provider disagrees with the current run (missing keys pass for back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. """ @@ -306,12 +255,20 @@ def load_resume_skip_keys( for field, expected in ( ("surface", surface), ("battery", battery), - ("model", model), ("driver", driver), + ("provider", provider), ): msg = _resume_field_mismatch(row, field=field, expected=expected) if msg: raise SystemExit(msg) + # New API rows keep both the requested ID (resume identity) and the + # provider-reported model that actually ran. Older rows only have model. + model_row = dict(row) + if model_row.get("requested_model"): + model_row["model"] = model_row["requested_model"] + msg = _resume_field_mismatch(model_row, field="model", expected=model) + if msg: + raise SystemExit(msg) # Meta / header rows: checked above, not part of resume key set. if is_meta_or_non_task_row(row): continue @@ -338,6 +295,7 @@ def make_run_meta_row( model: str | None, driver: str, git_sha: str, + provider: str | None = None, ts: str | None = None, ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" @@ -348,6 +306,7 @@ def make_run_meta_row( "battery": battery, "model": model, "driver": driver, + "provider": provider, "git_sha": git_sha, "ts": ts or datetime.now(timezone.utc).isoformat(), } @@ -395,7 +354,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "External MCP stdio server launch command (shlex-split), e.g. " "'/path/venv/bin/python -m plane_mcp stdio --v2'. Enables external mode: " "all tasks run (no surface skips) and mispick classification is disabled " - "(the foreign tool names have no overlay sets). CLI drivers only." + "(the foreign tool names have no overlay sets)." ), ) p.add_argument( @@ -408,12 +367,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p.add_argument( "--driver", type=str, - default="sdk", + default="api", choices=sorted(KNOWN_DRIVERS), help=( - "Agent backend: sdk | claude-cli | codex-cli | antigravity-cli | opencode-cli. Not required for --canary." + "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli " + "('sdk' is an alias for 'api'). Not required for --canary." ), ) + p.add_argument( + "--provider", + type=str, + default="anthropic", + choices=("anthropic", "openai"), + help="Model API provider for --driver api/sdk (default: anthropic).", + ) p.add_argument("--out", type=str, default=None, help="JSONL output path") p.add_argument( "--resume", @@ -475,190 +442,6 @@ def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: return 0 -async def run_agent_task( - *, - client: Any, - model_id: str, - task: dict[str, Any], - ctx: dict[str, Any], - workspace_slug: str, - surface: str = "full", - optimal_tools: set[str] | None = None, - alternate_tools: set[str] | None = None, -) -> dict[str, Any]: - """Run one task against a fresh stdio MCP server subprocess.""" - from anthropic.lib.tools.mcp import async_mcp_tool - from mcp import ClientSession - from mcp.client.stdio import StdioServerParameters, stdio_client - - project_name = ctx["project_name"] - system = _system_preamble(workspace_slug, project_name) - # strict: empty binder values / exceptions are infra_seed, not blank-ID prompts. - prompt = format_task_prompt(task, ctx, strict=True) - - server_params = StdioServerParameters( - command=sys.executable, - args=["-m", "plane_mcp", "stdio"], - env=stdio_server_env(surface=surface), - ) - - optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) - alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) - assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" - - calls: list[dict[str, Any]] = [] - # (call_idx, text, kind, is_error) buffered for post-loop count_tokens - pending_results: list[tuple[int, str, str, bool]] = [] - usage_per_iteration: list[dict[str, int]] = [] - final_message = None - iterations = 0 - result_pair_mismatch = False - wall_time_s = 0.0 - - async with stdio_client(server_params) as (read, write): - async with ClientSession(read, write) as mcp_client: - await mcp_client.initialize() - tools_result = await mcp_client.list_tools() - runner = client.beta.messages.tool_runner( - model=model_id, - max_tokens=MAX_TOKENS, - max_iterations=MAX_ITERATIONS, - system=system, - messages=[{"role": "user", "content": prompt}], - tools=[async_mcp_tool(t, mcp_client) for t in tools_result.tools], - ) - # wall_time: agent loop only (after list_tools, before subprocess teardown) - t0 = time.perf_counter() - try: - async for message in runner: - iterations += 1 - final_message = message - usage = getattr(message, "usage", None) - if usage is not None: - usage_per_iteration.append( - { - "in": getattr(usage, "input_tokens", 0) or 0, - "out": getattr(usage, "output_tokens", 0) or 0, - "cache_read": getattr(usage, "cache_read_input_tokens", 0) or 0, - "cache_write": getattr(usage, "cache_creation_input_tokens", 0) or 0, - } - ) - - # Map tool_use_id → call index for result pairing (not ordinal-only). - tool_use_by_id: dict[str, int] = {} - for block in message.content or []: - if getattr(block, "type", None) == "tool_use": - name = block.name - args = block.input if hasattr(block, "input") else {} - try: - args_chars = len(json.dumps(args, default=str)) - except Exception: - args_chars = len(str(args)) - call_rec = { - "tool": name, - "class": classify_call(name, optimal, alternate), - "args_chars": args_chars, - "result_tokens": None, - "result_chars": 0, - "result_kind": "text", - "is_error": False, - } - idx = len(calls) - calls.append(call_rec) - use_id = getattr(block, "id", None) - if use_id: - tool_use_by_id[str(use_id)] = idx - - # Never execute tools on a refusal-terminated turn (F2 / SDK guard). - if getattr(message, "stop_reason", None) == "refusal": - continue - - tool_response = await runner.generate_tool_call_response() - if tool_response is not None: - if isinstance(tool_response, dict): - blocks = tool_response.get("content") or [] - else: - blocks = getattr(tool_response, "content", None) or [] - result_blocks = [ - b - for b in blocks - if getattr(b, "type", None) == "tool_result" - or (isinstance(b, dict) and b.get("type") == "tool_result") - ] - matched_ids: set[str] = set() - for block in result_blocks: - if isinstance(block, dict): - is_error = bool(block.get("is_error")) - raw_content = block.get("content") - tool_use_id = block.get("tool_use_id") - else: - is_error = bool(getattr(block, "is_error", False)) - raw_content = getattr(block, "content", None) - tool_use_id = getattr(block, "tool_use_id", None) - text, kind = _tool_result_text(raw_content) - if tool_use_id is not None and str(tool_use_id) in tool_use_by_id: - idx = tool_use_by_id[str(tool_use_id)] - matched_ids.add(str(tool_use_id)) - else: - result_pair_mismatch = True - continue - calls[idx]["is_error"] = is_error - calls[idx]["result_kind"] = kind - if kind == "text": - calls[idx]["result_chars"] = len(text) - else: - calls[idx]["result_chars"] = len(str(raw_content)) - pending_results.append((idx, text, kind, is_error)) - if len(matched_ids) != len(tool_use_by_id): - result_pair_mismatch = True - finally: - wall_time_s = time.perf_counter() - t0 - - # Token-count tool results after the agent loop (must not pollute wall_time). - token_count_failures = 0 - for idx, text, kind, _is_error in pending_results: - if kind not in ("text", "mixed"): - calls[idx]["result_tokens"] = None - continue - counted = await _count_result_tokens(client, model_id, text) - calls[idx]["result_tokens"] = counted - if counted is None and text: - token_count_failures += 1 - - stop_reason = getattr(final_message, "stop_reason", None) if final_message else None - # Cap detection is stop_reason-aware only (F0/F3): a clean end_turn (or max_tokens, - # which the report already counts separately) on the 15th yield is not flagged here. - # Only runs that exhaust the iteration budget while still mid-tool-loop count. - hit_max_iterations = iterations >= MAX_ITERATIONS and stop_reason not in ( - "end_turn", - "max_tokens", - ) - - final_text = _extract_final_text(final_message) - errored = sum(1 for c in calls if c.get("is_error")) - alternate_n = sum(1 for c in calls if c["class"] == "alternate") - out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") - total_result_tokens = sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None) - cum_input = sum(u.get("in", 0) for u in usage_per_iteration) - - return { - "final_text": final_text, - "calls": calls, - "num_calls": len(calls), - "errored_calls": errored, - "alternate_calls": alternate_n, - "out_of_set_calls": out_of_set_n, - "total_result_tokens": total_result_tokens, - "usage_per_iteration": usage_per_iteration, - "cum_input_tokens": cum_input, - "wall_time_s": round(wall_time_s, 3), - "stop_reason": stop_reason, - "hit_max_iterations": hit_max_iterations, - "result_pair_mismatch": result_pair_mismatch, - "token_count_failures": token_count_failures, - } - - async def run_agent_task_via_driver( *, driver: Any, @@ -671,7 +454,7 @@ async def run_agent_task_via_driver( alternate_tools: set[str] | None = None, server_env: dict[str, str] | None = None, ) -> dict[str, Any]: - """Run one task through a CLI (or other) AgentDriver.""" + """Run one task through the selected AgentDriver.""" project_name = ctx["project_name"] system = _system_preamble(workspace_slug, project_name) prompt = format_task_prompt(task, ctx, strict=True) @@ -680,7 +463,7 @@ async def run_agent_task_via_driver( assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" mcp_env = stdio_server_env(surface=surface, extra=server_env) - # Drivers are sync (subprocess); run off the event loop thread. + # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. agent_run = await asyncio.to_thread( driver.run_task, prompt, @@ -695,7 +478,7 @@ async def run_agent_task_via_driver( optimal=optimal, alternate=alternate, classify=classify_call, - skip_result_tokens=True, + skip_result_tokens=agent_run.result_tokens_estimated is None, ) @@ -705,6 +488,7 @@ def _base_row( git_sha: str, surface: str, driver_name: str, + provider: str | None, model_id: str | None, task: dict[str, Any], rep: int, @@ -718,8 +502,10 @@ def _base_row( "battery": battery, "surface": surface, "driver": driver_name, + "provider": provider, "classification": classification, "model": model_id, + "requested_model": model_id, "task_id": task["id"], "author": task_author(task), "rep": rep, @@ -728,8 +514,12 @@ def _base_row( "skipped": None, "error": None, "error_class": None, + "final_text": "", "stop_reason": None, "hit_max_iterations": False, + "result_pair_mismatch": False, + "token_count_failures": 0, + "result_tokens_estimated": None, "calls": [], "num_calls": 0, "errored_calls": 0, @@ -749,7 +539,8 @@ async def run_live( reps: int, surface: str, out_path: Path, - driver_name: str = "sdk", + driver_name: str = "api", + provider: str = "anthropic", server_cmd: list[str] | None = None, server_env: dict[str, str] | None = None, resume: bool = False, @@ -764,22 +555,17 @@ async def run_live( ) return 2 - driver_name = (driver_name or "sdk").strip().lower() + driver_name = (driver_name or "api").strip().lower() if driver_name not in KNOWN_DRIVERS: print( f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", file=sys.stderr, ) return 2 - if external and driver_name == "sdk": - print( - f"error: --server-cmd requires a CLI driver (one of {sorted(KNOWN_DRIVERS - {'sdk'})})", - file=sys.stderr, - ) - return 2 - - use_sdk = driver_name == "sdk" - model_id = resolve_model_for_driver(driver_name, model_alias) + provider = (provider or "anthropic").strip().lower() + is_api_driver = driver_name in ("api", "sdk") + provider_id = provider if is_api_driver else None + model_id = resolve_model_for_driver(driver_name, model_alias, provider=provider) run_id = uuid.uuid4().hex git_sha = _git_sha() @@ -795,6 +581,7 @@ async def run_live( battery=battery, model=model_id, driver=driver_name, + provider=provider_id, ) except SystemExit as e: print(e, file=sys.stderr) @@ -808,6 +595,7 @@ async def run_live( battery=battery, model=model_id, driver=driver_name, + provider=provider_id, git_sha=git_sha, ) if maybe_write_run_meta(out_path, meta): @@ -816,24 +604,23 @@ async def run_live( plane, workspace_slug = make_plane_client() # User chose --driver explicitly: codex live is allowed (they own the quota). driver_kwargs: dict[str, Any] = {} + if is_api_driver: + driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) if driver_name == "codex-cli": driver_kwargs["allow_live"] = True - # --server-cmd must reach every CLI driver (not just Claude); otherwise we + # --server-cmd must reach every driver; otherwise we # silently benchmark the wrong server. if server_cmd is not None: - if use_sdk: - print("error: --server-cmd is incompatible with --driver sdk", file=sys.stderr) - return 2 driver_kwargs["server_command"] = server_cmd - cli_driver = None if use_sdk else get_driver(driver_name, **driver_kwargs) + driver = get_driver(driver_name, **driver_kwargs) print( - f"run_id={run_id} battery={battery} driver={driver_name} model={model_id} " + f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} model={model_id} " f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" ) print(f"writing {out_path}") - async def _one_client_scope(client: Any | None) -> None: + async def _run_tasks() -> None: with out_path.open("a", encoding="utf-8") as fh: for task in tasks: if external: @@ -858,6 +645,7 @@ async def _one_client_scope(client: Any | None) -> None: git_sha=git_sha, surface=surface, driver_name=driver_name, + provider=provider_id, model_id=model_id, task=task, rep=rep, @@ -902,34 +690,20 @@ async def _one_client_scope(client: Any | None) -> None: print(f" {task['id']} rep={rep} SKIPPED: {reason}") else: agent: dict[str, Any] | None = None - # Agent wrap: SDK bugs → infra_sdk; CLI raises → infra_cli. + # Agent wrap: API failures and CLI failures are infrastructure. # Contained CLI stops (timeout / error subtypes) return AgentRun. try: - if use_sdk: - assert client is not None - agent = await run_agent_task( - client=client, - model_id=model_id, - task=task, - ctx=ctx, - workspace_slug=workspace_slug, - surface=surface, - optimal_tools=surface_sets["optimal_tools"], - alternate_tools=surface_sets["alternate_tools"], - ) - else: - assert cli_driver is not None - agent = await run_agent_task_via_driver( - driver=cli_driver, - model_id=model_id, - task=task, - ctx=ctx, - workspace_slug=workspace_slug, - surface=surface, - optimal_tools=surface_sets["optimal_tools"], - alternate_tools=surface_sets["alternate_tools"], - server_env=server_env, - ) + agent = await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=ctx, + workspace_slug=workspace_slug, + surface=surface, + optimal_tools=surface_sets["optimal_tools"], + alternate_tools=surface_sets["alternate_tools"], + server_env=server_env, + ) except PromptBindError as exc: # Empty/missing seed IDs in the prompt — not an agent failure. row["success"] = False @@ -942,8 +716,12 @@ async def _one_client_scope(client: Any | None) -> None: ) agent = None except Exception as exc: - # SDK harness bugs must not look like CLI infra. - agent_err_class = "infra_sdk" if use_sdk else "infra_cli" + if driver_name == "sdk": + agent_err_class = "infra_sdk" + elif is_api_driver: + agent_err_class = "infra_api" + else: + agent_err_class = "infra_cli" row["success"] = False row["error"] = f"{type(exc).__name__}: {exc}" row["error_class"] = agent_err_class @@ -955,7 +733,7 @@ async def _one_client_scope(client: Any | None) -> None: agent = None if agent is not None: - row.update({k: agent[k] for k in agent if k != "final_text"}) + row.update(agent) if external: # Empty overlay sets would classify every call # out-of-set; null the counters instead. @@ -964,7 +742,7 @@ async def _one_client_scope(client: Any | None) -> None: # CLI infra stops: timeout + error subtypes except error_max_turns. stop_reason = agent.get("stop_reason") - if not use_sdk and is_infra_cli_stop_reason( + if driver_name.endswith("-cli") and is_infra_cli_stop_reason( str(stop_reason) if stop_reason is not None else None ): row["success"] = False @@ -1035,13 +813,7 @@ async def _one_client_scope(client: Any | None) -> None: fh.write(json.dumps(row, default=str) + "\n") fh.flush() - if use_sdk: - from anthropic import AsyncAnthropic - - async with AsyncAnthropic() as client: - await _one_client_scope(client) - else: - await _one_client_scope(None) + await _run_tasks() return 0 @@ -1169,7 +941,7 @@ def main(argv: list[str] | None = None) -> int: if args.canary: return asyncio.run(run_canary(tasks, surface=surface)) - driver_name = (getattr(args, "driver", None) or "sdk").strip().lower() + driver_name = (getattr(args, "driver", None) or "api").strip().lower() if driver_name not in KNOWN_DRIVERS: print( f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", @@ -1192,6 +964,7 @@ def main(argv: list[str] | None = None) -> int: surface=surface, out_path=out, driver_name=driver_name, + provider=args.provider, server_cmd=server_cmd, server_env=server_env or None, resume=bool(args.resume), diff --git a/pyproject.toml b/pyproject.toml index 8b301b02..d97f9dee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,9 @@ dev = [ "pytest>=7.0.0", "ruff>=0.1.0", ] -# Only the `sdk` eval driver needs this; the CLI drivers shell out to an agent. +# The default API eval provider uses the stable Anthropic Messages client. evals = [ - "anthropic[mcp]>=0.121.0", + "anthropic>=0.121.0", ] [project.scripts] diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py new file mode 100644 index 00000000..c24784b9 --- /dev/null +++ b/tests/test_evals_api_driver.py @@ -0,0 +1,531 @@ +"""Offline tests for the provider-generic API eval driver.""" + +from __future__ import annotations + +import copy +from collections import deque +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import Any + +from evals.drivers import agent_run_to_harness_dict +from evals.drivers.api import ( + AnthropicBackend, + ApiDriver, + OpenAIBackend, + ToolCall, + ToolResult, + ToolSpec, + Turn, +) + + +class FakeBackend: + provider = "fake" + model = "fake-requested" + actual_model = "fake-actual" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = deque(turns) + self.started: tuple[str | None, str, list[ToolSpec]] | None = None + self.added_results: list[list[ToolResult]] = [] + self.num_turns = 0 + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.started = (system, prompt, tools) + + def next_turn(self) -> Turn: + self.num_turns += 1 + if not self.turns: + raise AssertionError("driver requested an unexpected backend turn") + return self.turns.popleft() + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.added_results.append(results) + + +class FakeMcpSession: + def __init__(self, results: list[Any] | None = None) -> None: + self.results = deque(results or []) + self.initialized = False + self.called: list[tuple[str, dict[str, Any]]] = [] + + async def initialize(self) -> None: + self.initialized = True + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace( + name="lookup", + description="Look something up", + inputSchema={"type": "object", "properties": {"q": {"type": "string"}}}, + ), + SimpleNamespace( + name="write", + description="Write something", + inputSchema={"type": "object"}, + ), + ] + ) + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> Any: + self.called.append((name, arguments)) + if not self.results: + raise AssertionError(f"no fake result left for {name}") + return self.results.popleft() + + +def make_driver(backend: FakeBackend, session: FakeMcpSession) -> ApiDriver: + @asynccontextmanager + async def session_factory(_params): + yield session + + return ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + mcp_session_factory=session_factory, + ) + + +def run_driver(driver: ApiDriver, *, max_turns: int = 5): + return driver.run_task( + "do it", + {"SAFE": "1"}, + "fake-requested", + max_turns, + system="system", + ) + + +def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], + usage={"in": 10, "out": 2, "cache_read": 3, "cache_write": 1}, + stop_reason="tool_use", + ), + Turn( + text="", + tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], + usage={"in": 20, "out": 4, "cache_read": 6, "cache_write": 0}, + stop_reason="tool_use", + ), + Turn( + text="done", + tool_calls=[], + usage={"in": 30, "out": 6, "cache_read": 9, "cache_write": 0}, + stop_reason="end_turn", + ), + ] + ) + session = FakeMcpSession( + [ + {"content": [{"type": "text", "text": "first result"}], "isError": False}, + {"content": [{"type": "text", "text": "second"}], "isError": True}, + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert session.initialized is True + assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] + assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] + assert run.final_text == "done" + assert run.stopped_reason == "end_turn" + assert run.cum_input_tokens == 60 + assert run.usage_per_iteration == [ + {"in": 10, "out": 2, "cache_read": 3, "cache_write": 1}, + {"in": 20, "out": 4, "cache_read": 6, "cache_write": 0}, + {"in": 30, "out": 6, "cache_read": 9, "cache_write": 0}, + ] + assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] + assert [call["is_error"] for call in run.calls] == [False, True] + assert run.result_tokens_estimated is True + assert run.token_count_failures == 0 + assert run.provider == "fake" + assert run.model == "fake-actual" + + +def test_api_driver_refusal_records_calls_but_executes_nothing(): + backend = FakeBackend( + [ + Turn( + text="declined", + tool_calls=[ToolCall("write-1", "write", {"value": "x"})], + usage={"in": 1, "out": 1, "cache_read": 0, "cache_write": 0}, + stop_reason="refusal", + ) + ] + ) + session = FakeMcpSession() + + run = run_driver(make_driver(backend, session)) + + assert [call["tool"] for call in run.calls] == ["write"] + assert session.called == [] + assert backend.added_results == [] + assert run.stopped_reason == "refusal" + assert run.hit_max_turns is False + + +def test_api_driver_pairs_results_by_id_not_ordinal(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason="tool_use", + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + ] + ) + # The fake session deliberately returns tagged results in reverse ID order. + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="a", text="A"), + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert [call["result_chars"] for call in run.calls] == [1, 4] + assert run.result_pair_mismatch is False + + +def test_api_driver_flags_result_id_mismatch(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason="tool_use", + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="unknown", text="lost"), + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert run.result_pair_mismatch is True + assert [call["result_chars"] for call in run.calls] == [0, 4] + + +def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason="tool_use", + ), + Turn(text="must not be read", tool_calls=[], usage=None, stop_reason="end_turn"), + ] + ) + session = FakeMcpSession([ToolResult(call_id="a", text="result")]) + + run = run_driver(make_driver(backend, session), max_turns=1) + + assert session.called == [("lookup", {"q": "a"})] + assert len(backend.added_results) == 1 + assert backend.num_turns == 1 + assert run.hit_max_turns is True + assert run.stopped_reason == "tool_use" + + +def test_api_driver_clean_end_on_last_iteration_is_not_capped(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn")]) + + run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) + + assert run.hit_max_turns is False + assert run.stopped_reason == "end_turn" + + +def test_api_driver_uses_optional_backend_token_counter(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason="tool_use", + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + ] + ) + backend.count_tokens = lambda text: len(text) + 10 + + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) + + assert run.calls[0]["result_tokens"] == 13 + assert run.result_tokens_estimated is False + assert run.token_count_failures == 0 + + +def test_api_driver_maps_every_legacy_row_field(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage={"in": 4, "out": 1, "cache_read": 0, "cache_write": 0}, + stop_reason="tool_use", + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + ] + ) + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) + row = agent_run_to_harness_dict( + run, + optimal={"lookup"}, + alternate=set(), + classify=lambda tool, optimal, alternate: ( + "optimal" if tool in optimal else "alternate" if tool in alternate else "out_of_set" + ), + skip_result_tokens=False, + ) + + required = { + "final_text", + "calls", + "num_calls", + "errored_calls", + "alternate_calls", + "out_of_set_calls", + "total_result_tokens", + "usage_per_iteration", + "cum_input_tokens", + "wall_time_s", + "stop_reason", + "hit_max_iterations", + "result_pair_mismatch", + "token_count_failures", + } + assert required <= row.keys() + assert { + "tool", + "class", + "args_chars", + "result_tokens", + "result_chars", + "result_kind", + "is_error", + } <= row["calls"][0].keys() + assert row["calls"][0]["result_chars"] == 5 + assert row["calls"][0]["result_tokens"] == 2 + assert row["result_tokens_estimated"] is True + assert row["provider"] == "fake" + assert row["model"] == "fake-actual" + assert row["requested_model"] == "fake-requested" + + +class FakeAnthropicMessages: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + +def test_anthropic_backend_translates_tools_turns_and_results(): + responses = [ + { + "model": "claude-actual", + "content": [ + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu-1", "name": "lookup", "input": {"q": "x"}}, + ], + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "cache_read_input_tokens": 3, + "cache_creation_input_tokens": 4, + }, + "stop_reason": "tool_use", + }, + { + "model": "claude-actual", + "content": [{"type": "text", "text": "done"}], + "usage": {"input_tokens": 20, "output_tokens": 5}, + "stop_reason": "end_turn", + }, + ] + messages = FakeAnthropicMessages(responses) + backend = AnthropicBackend( + "claude-requested", + max_tokens=123, + client=SimpleNamespace(messages=messages), + ) + tool = ToolSpec("lookup", "Look up", {"type": "object", "required": ["q"]}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("toolu-1", "value", is_error=True)]) + second = backend.next_turn() + + assert messages.requests[0]["system"] == "system" + assert messages.requests[0]["max_tokens"] == 123 + assert messages.requests[0]["tools"] == [ + {"name": "lookup", "description": "Look up", "input_schema": {"type": "object", "required": ["q"]}} + ] + assert first.tool_calls == [ToolCall("toolu-1", "lookup", {"q": "x"})] + assert first.usage == {"in": 10, "out": 2, "cache_read": 3, "cache_write": 4} + replay = messages.requests[1]["messages"] + assert replay[1] == {"role": "assistant", "content": responses[0]["content"]} + assert replay[2] == { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu-1", + "content": "value", + "is_error": True, + } + ], + } + assert second.text == "done" + assert backend.actual_model == "claude-actual" + + +class FakeOpenAICompletions: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + +def test_openai_backend_translates_tools_calls_and_tool_messages(): + responses = [ + { + "model": "gpt-actual", + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + } + ], + }, + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 3, + "prompt_tokens_details": {"cached_tokens": 5}, + }, + }, + { + "model": "gpt-actual", + "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], + "usage": {"prompt_tokens": 20, "completion_tokens": 4}, + }, + ] + completions = FakeOpenAICompletions(responses) + client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) + backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("call-1", "value")]) + second = backend.next_turn() + + first_request = completions.requests[0] + assert first_request["max_completion_tokens"] == 321 + assert first_request["messages"] == [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "prompt"}, + ] + assert first_request["tools"] == [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + } + ] + assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] + assert first.stop_reason == "tool_use" + assert first.usage == {"in": 12, "out": 3, "cache_read": 5, "cache_write": 0} + second_messages = completions.requests[1]["messages"] + assert second_messages[2] == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + } + ], + } + assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} + assert second.text == "done" + assert second.stop_reason == "end_turn" + assert backend.actual_model == "gpt-actual" + + +def test_openai_backend_normalizes_refusal_for_driver_guard(): + completions = FakeOpenAICompletions( + [ + { + "model": "gpt", + "choices": [ + { + "finish_reason": "content_filter", + "message": { + "content": None, + "refusal": "declined", + "tool_calls": [ + { + "id": "danger", + "type": "function", + "function": {"name": "write", "arguments": "{}"}, + } + ], + }, + } + ], + "usage": None, + } + ] + ) + backend = OpenAIBackend( + "gpt", + max_tokens=10, + client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), + ) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason == "refusal" + assert turn.text == "declined" + assert turn.tool_calls == [ToolCall("danger", "write", {})] diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index 2b90cc92..fe23eb62 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -16,6 +16,7 @@ from evals.drivers import ( KNOWN_DRIVERS, AgentRun, + ApiDriver, ClaudeCliDriver, CodexCliDriver, agent_run_to_harness_dict, @@ -766,11 +767,12 @@ def test_agent_run_to_harness_dict_does_not_guess_usage_total(): def test_known_drivers(): - assert KNOWN_DRIVERS == {"sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + assert KNOWN_DRIVERS == {"api", "sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} -def test_get_driver_sdk_is_none(): - assert get_driver("sdk") is None +def test_get_driver_api_and_sdk_alias(): + assert isinstance(get_driver("api"), ApiDriver) + assert isinstance(get_driver("sdk"), ApiDriver) assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) assert isinstance(get_driver("codex-cli"), CodexCliDriver) @@ -779,7 +781,8 @@ def test_parse_args_accepts_driver(): a = parse_args(["--driver", "claude-cli", "--dry-run"]) assert a.driver == "claude-cli" b = parse_args(["--dry-run"]) - assert b.driver == "sdk" + assert b.driver == "api" + assert b.provider == "anthropic" def test_stdio_env_still_works_for_cli_drivers(monkeypatch): diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 73163691..4e5f07ea 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -972,3 +972,4 @@ async def verify_ok(plane, ctx, run): assert new_r2["task_id"] == "R2" assert new_r2["success"] is True assert new_r2["error_class"] is None + assert new_r2["final_text"] == "done" diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index 1f7a05dc..3fabfde2 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -978,6 +978,8 @@ def test_resolve_model_for_driver_qualification(): # Free-form passthrough assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" assert resolve_model_for_driver("sdk", "sonnet") == "claude-sonnet-5" + assert resolve_model_for_driver("api", "haiku") == "claude-haiku-4-5" + assert resolve_model_for_driver("api", "sonnet", provider="openai") == "gpt-5" def test_ensure_proxy_pythonpath_injects_repo(): diff --git a/tests/test_evals_surface.py b/tests/test_evals_surface.py index f44f34fe..32f8d907 100644 --- a/tests/test_evals_surface.py +++ b/tests/test_evals_surface.py @@ -131,16 +131,6 @@ def test_skip_path_no_network(monkeypatch): import tempfile from pathlib import Path - # Avoid importing anthropic for skip-only path: patch AsyncAnthropic too - class _FakeAnthro: - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return None - - monkeypatch.setattr("anthropic.AsyncAnthropic", lambda: _FakeAnthro()) - with tempfile.TemporaryDirectory() as td: out = Path(td) / "out.jsonl" rc = asyncio.run( From 4afe65518b2ef1e7a0bdddbbb47a1be10075a516 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 01:42:49 +0530 Subject: [PATCH 04/93] Report response-token cost for every driver, and label estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the in-process driver could report what a tool response costs in context, because it was the sole path holding the result text; CLI rows carried a "no API key" marker instead of a number. That silently limited the metric to one driver, and the metric is a large part of why the harness exists. Both paths now derive tokens from one shared estimator, so they cannot drift, and rows say when a count is estimated rather than measured — report.py labels a column estimated, measured, or mixed instead of presenting all three alike. Exact CLI-side counting is available but off by default: the proxy can retain serialized tool-result text for a real tokenizer to count, at the cost of writing live workspace data into the sidecar. For comparing surfaces the default estimate is monotonic in the quantity being compared, so the precision is rarely worth that trade; the README says so where the flag is documented. The proxy stays stdlib-only — it runs inside the server's process tree with a scrubbed PYTHONPATH, so tokenizing happens in analysis code, never there. Co-Authored-By: Claude Fable 5 --- evals/DESIGN.md | 4 ++ evals/README.md | 13 ++++- evals/drivers/antigravity.py | 9 +++- evals/drivers/api/driver.py | 15 +++--- evals/drivers/base.py | 78 +++++++++++++++++++++++------- evals/drivers/claude.py | 9 +++- evals/drivers/codex.py | 9 +++- evals/drivers/opencode.py | 9 +++- evals/drivers/sidecar.py | 30 +++++++----- evals/proxy.py | 50 ++++++++++++------- evals/report.py | 63 ++++++++++++++++++++++-- evals/run.py | 13 ++++- evals/token_counting.py | 75 +++++++++++++++++++++++++++++ tests/test_evals_api_driver.py | 8 +++- tests/test_evals_drivers.py | 88 ++++++++++++++++++++++++++++++++-- tests/test_evals_proxy.py | 70 +++++++++++++++++++++++++++ tests/test_evals_report_ops.py | 51 ++++++++++++++++++++ 17 files changed, 523 insertions(+), 71 deletions(-) create mode 100644 evals/token_counting.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index dce95fe2..7c8e3b24 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -71,6 +71,10 @@ model, so `result_chars` is always exact. A backend may expose a token counter; driver uses a deterministic character estimate and sets `result_tokens_estimated: true` on the row. No provider token-count endpoint is required per tool result. +CLI rows use the same shared character estimator over proxy-recorded `result_chars`. Optional +payload recording permits local tokenizer counting in the parent harness, but stays off by +default; the stdlib-only proxy does not import tokenizers. + Rules: - Run these counts **after** the agent loop finishes, not inline — they must not pollute `wall_time_s`. Buffer the raw result strings during the run, count at the end. diff --git a/evals/README.md b/evals/README.md index df1298a3..07587bd5 100644 --- a/evals/README.md +++ b/evals/README.md @@ -89,8 +89,17 @@ calls are counted from the wire rather than from whatever the agent claims it di The API driver executes MCP calls itself, records exact result character counts, and sizes result tokens without making a provider request per result. A backend may supply a token counter; otherwise rows set `result_tokens_estimated: true` and use a deterministic -character-based estimate. CLI drivers retain null result-token counts because they do not -always expose complete tool result text. +character-based estimate. CLI drivers use that same shared estimate from the result character +counts in the recording sidecar, so every driver reports response-token cost and estimated +counts are explicitly marked. + +Exact CLI-side counting is opt-in with `--record-result-payloads`. It makes the proxy retain +the serialized tool-result text long enough for the harness to count it with a locally +importable tokenizer (`tiktoken`/`cl100k_base`); if that tokenizer is unavailable, the harness +falls back to the same marked estimate. The default stays **off** because payloads contain live +workspace data and bloat the sidecar. For comparing tool surfaces, the default chars-derived +estimate is monotonic in the thing being compared anyway. Do not enable payload recording by +habit; use it only when the more sensitive, larger sidecar is justified. ### Reading results diff --git a/evals/drivers/antigravity.py b/evals/drivers/antigravity.py index 0fb0d4dc..d7758ca8 100644 --- a/evals/drivers/antigravity.py +++ b/evals/drivers/antigravity.py @@ -130,12 +130,14 @@ def __init__( runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, server_command: list[str] | None = None, use_proxy: bool = True, + record_result_payloads: bool = False, ) -> None: self.agy_bin = agy_bin self.python_bin = python_bin or sys.executable self._runner = runner or run_cli_subprocess self.server_command = list(server_command) if server_command else None self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads def run_task( self, @@ -161,7 +163,12 @@ def run_task( else: real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + wrapped = proxy_wrap_server_command( + real_cmd, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) server_cmd, server_args = wrapped[0], wrapped[1:] child_env_plane = ensure_proxy_pythonpath(child_env_plane) else: diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 236dd3b4..eaf119f0 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -19,6 +19,7 @@ from evals.drivers.api.backend import ModelBackend, ToolResult, ToolSpec from evals.drivers.api.openai import OpenAIBackend from evals.drivers.base import AgentRun +from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens DEFAULT_MAX_TOKENS = 8192 KNOWN_API_PROVIDERS = frozenset({"anthropic", "openai"}) @@ -27,13 +28,6 @@ McpSessionFactory = Callable[[StdioServerParameters], Any] -def estimate_tokens(text: str) -> int: - """Estimate token count from recorded text without a provider request.""" - if not text: - return 0 - return max(1, (len(text) + 3) // 4) - - def tool_spec_from_mcp(tool: Any) -> ToolSpec: """Translate an MCP list-tools entry into a neutral tool specification.""" if isinstance(tool, dict): @@ -301,6 +295,7 @@ async def _run_task( result_tokens_estimated = False for idx, result_text in pending_results: counted: int | None = None + count_estimated = False if callable(counter): try: raw_count = counter(result_text) @@ -310,9 +305,12 @@ async def _run_task( except Exception: token_count_failures += 1 if counted is None: - counted = estimate_tokens(result_text) + counted = estimate_result_tokens(len(result_text)) + count_estimated = True result_tokens_estimated = True calls[idx]["result_tokens"] = counted + calls[idx]["result_tokens_estimated"] = count_estimated + calls[idx]["result_token_count_method"] = TOKEN_ESTIMATE_METHOD if count_estimated else "backend" usage_total = { "input_tokens": sum(item.get("in", 0) for item in usage_per_iteration), @@ -348,7 +346,6 @@ async def _run_task( "ApiDriver", "BackendFactory", "McpSessionFactory", - "estimate_tokens", "tool_result_from_mcp", "tool_spec_from_mcp", ] diff --git a/evals/drivers/base.py b/evals/drivers/base.py index 6ece231b..cbbf0644 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -9,6 +9,12 @@ from pathlib import Path from typing import Any, Protocol +from evals.token_counting import ( + TOKEN_ESTIMATE_METHOD, + count_result_text_tokens, + estimate_result_tokens, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent # mcp__plane__list_work_items → list_work_items @@ -43,8 +49,9 @@ class AgentRun: cum_input_tokens: int | None = None result_pair_mismatch: bool = False token_count_failures: int = 0 - # None means result token sizing was skipped (CLI drivers). False means a - # backend counter was used; True means at least one result used estimation. + # False means a tokenizer/backend counter was used for every result; True + # means at least one result used the shared character estimate. None lets + # the common row mapper determine the status from the recorded calls. result_tokens_estimated: bool | None = None provider: str | None = None model: str | None = None @@ -154,7 +161,6 @@ def agent_run_to_harness_dict( optimal: set[str], alternate: set[str], classify: Callable[[str, set[str], set[str]], str], - skip_result_tokens: bool = True, ) -> dict[str, Any]: """Map an ``AgentRun`` onto the dict shape expected by ``run_live`` rows. @@ -170,7 +176,9 @@ def agent_run_to_harness_dict( plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) client_src = list(run.client_tool_calls) + client_extra + is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" calls: list[dict[str, Any]] = [] + local_token_count_failures = 0 for c in plane_src: tool = c.get("tool") or "" args = c.get("args") or {} @@ -178,14 +186,37 @@ def agent_run_to_harness_dict( args_chars = len(json.dumps(args, default=str)) except Exception: args_chars = len(str(args)) + result_chars = int(c["result_chars"]) if c.get("result_chars") is not None else 0 + result_tokens = c.get("result_tokens") + estimated = c.get("result_tokens_estimated") + count_method = c.get("result_token_count_method") + if result_tokens is not None: + result_tokens = int(result_tokens) + if estimated is None: + estimated = bool(run.result_tokens_estimated) + if count_method is None: + count_method = TOKEN_ESTIMATE_METHOD if estimated else "backend" + elif isinstance(c.get("result_text"), str): + count = count_result_text_tokens(c["result_text"]) + result_tokens = count.value + estimated = count.estimated + count_method = count.method + local_token_count_failures += int(count.tokenizer_failed) + else: + result_tokens = estimate_result_tokens(result_chars) + estimated = True + count_method = TOKEN_ESTIMATE_METHOD + rec: dict[str, Any] = { "tool": tool, "class": classify(str(tool), optimal, alternate), "args_chars": args_chars, - "result_tokens": None if skip_result_tokens else c.get("result_tokens"), - "result_chars": int(c["result_chars"]) if c.get("result_chars") is not None else 0, + "result_tokens": result_tokens, + "result_chars": result_chars, "result_kind": str(c.get("result_kind") or "text"), "is_error": bool(c.get("is_error")), + "result_tokens_estimated": bool(estimated), + "result_token_count_method": str(count_method), } if c.get("duration_ms") is not None: rec["duration_ms"] = c["duration_ms"] @@ -193,8 +224,6 @@ def agent_run_to_harness_dict( # tool choice — keep it (args content is otherwise not persisted). if isinstance(args, dict) and isinstance(args.get("action"), str): rec["action"] = args["action"] - if skip_result_tokens: - rec["result_tokens_skipped"] = "CLI driver does not expose complete result text" calls.append(rec) client_tool_calls: list[dict[str, Any]] = [] @@ -225,7 +254,6 @@ def agent_run_to_harness_dict( # CLI path: never write misleading cum_input_tokens from uncached-only field. # usage_total is driver-owned — do not re-derive it here (Claude vs Codex # shapes differ; a generic Claude rebuild mislabels other vendors). - is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" usage_total = run.usage_total if run.usage_per_iteration: @@ -236,7 +264,7 @@ def agent_run_to_harness_dict( else sum(item.get("in", 0) for item in usage_per_iteration) ) cum_reason = None - elif is_cli and skip_result_tokens: + elif is_cli: cum_input: int | None = None cum_reason: str | None = ( "CLI driver: Claude usage.input_tokens is uncached-only; " @@ -250,6 +278,25 @@ def agent_run_to_harness_dict( if run.usage and run.usage_scope == "iteration": pass + estimated_states = [bool(c["result_tokens_estimated"]) for c in calls] + if estimated_states: + result_tokens_estimated = any(estimated_states) + result_tokens_mode = ( + "estimated" if all(estimated_states) else "measured" if not any(estimated_states) else "mixed" + ) + else: + result_tokens_estimated = ( + bool(run.result_tokens_estimated) if run.result_tokens_estimated is not None else is_cli + ) + result_tokens_mode = "estimated" if result_tokens_estimated else "measured" + + count_methods = {str(c["result_token_count_method"]) for c in calls} + if not count_methods: + result_token_count_method = "none" + elif len(count_methods) == 1: + result_token_count_method = next(iter(count_methods)) + else: + result_token_count_method = "mixed" result = { "final_text": run.final_text, "calls": calls, @@ -259,9 +306,7 @@ def agent_run_to_harness_dict( "errored_calls": errored, "alternate_calls": alternate_n, "out_of_set_calls": out_of_set_n, - "total_result_tokens": 0 - if skip_result_tokens - else sum(c["result_tokens"] or 0 for c in calls if c.get("result_tokens") is not None), + "total_result_tokens": sum(int(c["result_tokens"]) for c in calls), "usage_per_iteration": usage_per_iteration, "cum_input_tokens": cum_input, "cum_input_tokens_reason": cum_reason, @@ -269,15 +314,14 @@ def agent_run_to_harness_dict( "stop_reason": stop_reason, "hit_max_iterations": hit_max, "result_pair_mismatch": run.result_pair_mismatch, - "token_count_failures": run.token_count_failures, - "result_tokens_estimated": run.result_tokens_estimated, + "token_count_failures": run.token_count_failures + local_token_count_failures, + "result_tokens_estimated": result_tokens_estimated, + "result_tokens_mode": result_tokens_mode, + "result_token_count_method": result_token_count_method, "usage_scope": run.usage_scope, "call_source": run.call_source, "driver_raw_ref": run.raw_ref, "driver_notes": list(run.notes), - "result_tokens_skipped_reason": ( - "CLI driver: complete tool result text is unavailable; skipped" if skip_result_tokens else None - ), "usage": run.usage, "usage_total": usage_total, } diff --git a/evals/drivers/claude.py b/evals/drivers/claude.py index 6e7a0a29..cb810f6a 100644 --- a/evals/drivers/claude.py +++ b/evals/drivers/claude.py @@ -299,6 +299,7 @@ def __init__( runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, server_command: list[str] | None = None, use_proxy: bool = True, + record_result_payloads: bool = False, ) -> None: self.claude_bin = claude_bin self.python_bin = python_bin or sys.executable @@ -309,6 +310,7 @@ def __init__( # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. self.server_command = list(server_command) if server_command else None self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads def run_task( self, @@ -335,7 +337,12 @@ def run_task( else: real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + wrapped = proxy_wrap_server_command( + real_cmd, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) server_cmd, server_args = wrapped[0], wrapped[1:] child_env = ensure_proxy_pythonpath(child_env) else: diff --git a/evals/drivers/codex.py b/evals/drivers/codex.py index 01ffa440..a179ce00 100644 --- a/evals/drivers/codex.py +++ b/evals/drivers/codex.py @@ -257,6 +257,7 @@ def __init__( allow_live: bool = False, server_command: list[str] | None = None, use_proxy: bool = True, + record_result_payloads: bool = False, ) -> None: self.codex_bin = codex_bin self.python_bin = python_bin or sys.executable @@ -264,6 +265,7 @@ def __init__( self.allow_live = allow_live self.server_command = list(server_command) if server_command else None self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads def run_task( self, @@ -292,7 +294,12 @@ def run_task( else: real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] if self.use_proxy: - wrapped = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + wrapped = proxy_wrap_server_command( + real_cmd, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) server_cmd, server_args = wrapped[0], wrapped[1:] child_env = ensure_proxy_pythonpath(child_env) else: diff --git a/evals/drivers/opencode.py b/evals/drivers/opencode.py index 17a44a97..a007919f 100644 --- a/evals/drivers/opencode.py +++ b/evals/drivers/opencode.py @@ -72,12 +72,14 @@ def __init__( runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, server_command: list[str] | None = None, use_proxy: bool = True, + record_result_payloads: bool = False, ) -> None: self.opencode_bin = opencode_bin self.python_bin = python_bin or sys.executable self._runner = runner or run_cli_subprocess self.server_command = list(server_command) if server_command else None self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads def run_task( self, @@ -103,7 +105,12 @@ def run_task( else: real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] if self.use_proxy: - launch = proxy_wrap_server_command(real_cmd, sidecar_path=sidecar, python_bin=self.python_bin) + launch = proxy_wrap_server_command( + real_cmd, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) child_env_plane = ensure_proxy_pythonpath(child_env_plane) else: launch = real_cmd diff --git a/evals/drivers/sidecar.py b/evals/drivers/sidecar.py index 6bb60bcb..2a2d1b03 100644 --- a/evals/drivers/sidecar.py +++ b/evals/drivers/sidecar.py @@ -17,10 +17,14 @@ def proxy_wrap_server_command( *, sidecar_path: Path, python_bin: str | None = None, + record_result_payloads: bool = False, ) -> list[str]: """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" py = python_bin or sys.executable - return [py, "-m", "evals.proxy", "--log", str(sidecar_path), "--", *real_command] + command = [py, "-m", "evals.proxy", "--log", str(sidecar_path)] + if record_result_payloads: + command.append("--record-result-payloads") + return [*command, "--", *real_command] def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: @@ -91,17 +95,19 @@ def load_proxy_sidecar( tool = row.get("tool") if not tool: continue - calls.append( - { - "tool": str(tool), - "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), - "origin": "plane", - "is_error": bool(row.get("is_error")), - "result_chars": int(row.get("result_chars") or 0), - "duration_ms": row.get("duration_ms"), - "seq": row.get("seq"), - } - ) + call = { + "tool": str(tool), + "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), + "origin": "plane", + "is_error": bool(row.get("is_error")), + "result_chars": int(row.get("result_chars") or 0), + "duration_ms": row.get("duration_ms"), + "seq": row.get("seq"), + } + # Optional in new sidecars; old payload-free rows remain valid. + if isinstance(row.get("result_text"), str): + call["result_text"] = row["result_text"] + calls.append(call) # Score order must match request seq, not response-append order. calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) diff --git a/evals/proxy.py b/evals/proxy.py index 57f6ae2c..df73fb14 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -1,7 +1,7 @@ """Stdio MCP recording proxy — byte-faithful JSON-RPC relay with sidecar call log. Usage: - python -m evals.proxy --log SIDECAR.jsonl -- + python -m evals.proxy --log SIDECAR.jsonl [--record-result-payloads] -- Spawns the target as a child, relays parent stdin → child stdin and child stdout → parent stdout as raw bytes (byte-faithful; does not re-serialize). @@ -47,6 +47,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Sidecar JSONL path for recorded tool calls + proxy_meta summary", ) + p.add_argument( + "--record-result-payloads", + action="store_true", + help="Also store serialized tool-result text (off by default; may contain workspace data)", + ) p.add_argument( "command", nargs=argparse.REMAINDER, @@ -116,8 +121,9 @@ class SidecarRecorder: last sidecar line even if daemon pumps keep running briefly. """ - def __init__(self, log_path: Path) -> None: + def __init__(self, log_path: Path, *, record_result_payloads: bool = False) -> None: self.log_path = log_path + self.record_result_payloads = record_result_payloads self._lock = threading.Lock() self._pending: dict[Any, dict[str, Any]] = {} self._seq = 0 @@ -216,19 +222,20 @@ def on_server_message(self, obj: dict[str, Any]) -> None: is_error = bool(result.get("isError") or result.get("is_error")) result_payload = result try: - result_chars = len(json.dumps(result_payload, default=str, ensure_ascii=False)) + result_text = json.dumps(result_payload, default=str, ensure_ascii=False) except Exception: - result_chars = len(str(result_payload)) - self._append( - { - "tool": pending["tool"], - "args": pending["args"], - "is_error": is_error, - "result_chars": result_chars, - "duration_ms": duration_ms, - "seq": pending["seq"], - } - ) + result_text = str(result_payload) + row = { + "tool": pending["tool"], + "args": pending["args"], + "is_error": is_error, + "result_chars": len(result_text), + "duration_ms": duration_ms, + "seq": pending["seq"], + } + if self.record_result_payloads: + row["result_text"] = result_text + self._append(row) def write_meta(self) -> None: """Write proxy_meta as the last row and seal the sidecar (atomic under lock).""" @@ -390,7 +397,12 @@ def reap_timeout(deadline_at: float | None, floor: float = 0.1) -> float: return max(floor, _remaining(deadline_at)) -def run_proxy(command: list[str], log_path: Path) -> int: +def run_proxy( + command: list[str], + log_path: Path, + *, + record_result_payloads: bool = False, +) -> int: """Spawn ``command`` as the real MCP server and relay with recording. Returns the child's exit code (or 1 on spawn failure). Guarantees @@ -398,7 +410,7 @@ def run_proxy(command: list[str], log_path: Path) -> int: crash paths. Pump threads are daemon so a blocked write cannot hold the process past the shutdown deadline. """ - recorder = SidecarRecorder(log_path) + recorder = SidecarRecorder(log_path, record_result_payloads=record_result_payloads) child: subprocess.Popen[bytes] | None = None # Scrub repo PYTHONPATH so the real server does not import from this tree. child_env = scrub_child_pythonpath() @@ -580,7 +592,11 @@ def main(argv: list[str] | None = None) -> int: # Already a session leader, or platform forbids setsid — continue. pass args = parse_args(argv) - return run_proxy(list(args.command), Path(args.log)) + return run_proxy( + list(args.command), + Path(args.log), + record_result_payloads=bool(args.record_result_payloads), + ) if __name__ == "__main__": diff --git a/evals/report.py b/evals/report.py index 177724f6..fceb15df 100644 --- a/evals/report.py +++ b/evals/report.py @@ -20,6 +20,7 @@ from evals.tasks import TASKS_BY_ID DedupeMode = Literal["latest", "none"] +ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: @@ -82,6 +83,32 @@ def _iqr(xs: list[float]) -> tuple[float | None, float | None, float | None]: return (_percentile(xs, 0.25), _median(xs), _percentile(xs, 0.75)) +def result_tokens_mode(rows: list[dict[str, Any]]) -> ResultTokensMode: + """Classify token counts without treating unmarked legacy data as measured.""" + labels: set[str] = set() + for row in rows: + row_estimated = row.get("result_tokens_estimated") + for call in row.get("calls") or []: + if call.get("result_tokens") is None: + continue + estimated = call.get("result_tokens_estimated", row_estimated) + if estimated is True: + labels.add("estimated") + elif estimated is False: + labels.add("measured") + else: + labels.add("unlabeled") + if not labels: + return "unavailable" + if labels == {"estimated"}: + return "estimated" + if labels == {"measured"}: + return "measured" + if "unlabeled" in labels: + return "unlabeled" + return "mixed" + + def is_meta_row(row: dict[str, Any]) -> bool: """True for run-header meta lines (or any row without a task_id).""" if row.get("row_type") == "meta": @@ -230,6 +257,7 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: "infra_err": infra_err_by_task.get(task_id, 0), "med_result_tokens": _median(result_tokens), "p95_result_tokens": _percentile(result_tokens, 0.95), + "result_tokens_mode": result_tokens_mode(trs), "med_cum_input": _median(cum_inputs), } agg_lo, agg_hi = wilson_interval(total_k, total_n) if total_n else (0.0, 0.0) @@ -239,6 +267,7 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: "aggregate_n": total_n, "aggregate_wilson_lo": agg_lo, "aggregate_wilson_hi": agg_hi, + "result_tokens_mode": result_tokens_mode([r for trs in by_task.values() for r in trs]), } return out @@ -249,9 +278,27 @@ def _fmt(x: float | None, digits: int = 1) -> str: return f"{x:.{digits}f}" +def _result_tokens_marker(mode: str) -> str: + return {"estimated": "~", "mixed": "*", "unlabeled": "?"}.get(mode, "") + + +def _fmt_result_tokens(x: float | None, mode: str) -> str: + value = _fmt(x, 0) + if value == "-": + return value + return f"{_result_tokens_marker(mode)}{value}" + + def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: meta = summary.get("_meta") or {} print(title) + token_mode = str(meta.get("result_tokens_mode") or "unavailable") + if token_mode == "estimated": + print("result-token columns marked ~: entirely estimated from result characters") + elif token_mode == "mixed": + print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") + elif token_mode == "unlabeled": + print("result-token columns marked ?: include legacy values with unknown measurement status") if meta.get("infra_errors"): print(f"infra errors: {meta['infra_errors']}") agg_n = int(meta.get("aggregate_n") or 0) @@ -263,18 +310,23 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: print(f"aggregate success: {agg_k}/{agg_n} ({rate:.1%}) Wilson95 [{alo:.2f},{ahi:.2f}]") # Show min/med/max call columns when any task has n>1. show_var = any(s.get("n", 0) > 1 for tid, s in summary.items() if tid != "_meta") + token_marker = _result_tokens_marker(token_mode) + med_rtok_header = f"med_rtok{token_marker}" + p95_rtok_header = f"p95_rtok{token_marker}" if show_var: header = ( f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " f"{'IQR':>11} {'mispick':>8} {'err':>4} " - f"{'capped':>6} {'h_err':>5} {'i_err':>5} {'med_rtok':>8} {'p95_rtok':>8} {'med_cum_in':>10}" + f"{'capped':>6} {'h_err':>5} {'i_err':>5} {med_rtok_header:>9} {p95_rtok_header:>9} " + f"{'med_cum_in':>10}" ) else: header = ( f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " f"{'med_calls':>9} {'opt':>4} {'IQR':>11} {'mispick':>8} {'err':>4} " - f"{'capped':>6} {'h_err':>5} {'i_err':>5} {'med_rtok':>8} {'p95_rtok':>8} {'med_cum_in':>10}" + f"{'capped':>6} {'h_err':>5} {'i_err':>5} {med_rtok_header:>9} {p95_rtok_header:>9} " + f"{'med_cum_in':>10}" ) print(header) print("-" * len(header)) @@ -284,6 +336,7 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: wilson = f"[{s['wilson_lo']:.2f},{s['wilson_hi']:.2f}]" iqr = f"{_fmt(s['calls_q1'])}-{_fmt(s['calls_q3'])}" opt = s["optimal_calls"] if s["optimal_calls"] is not None else "-" + task_token_mode = str(s.get("result_tokens_mode") or "unavailable") if show_var: print( f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " @@ -291,7 +344,8 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: f"{opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " f"{s.get('infra_err', 0):>5} " - f"{_fmt(s['med_result_tokens'], 0):>8} {_fmt(s['p95_result_tokens'], 0):>8} " + f"{_fmt_result_tokens(s['med_result_tokens'], task_token_mode):>9} " + f"{_fmt_result_tokens(s['p95_result_tokens'], task_token_mode):>9} " f"{_fmt(s['med_cum_input'], 0):>10}" ) else: @@ -300,7 +354,8 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: f"{_fmt(s['med_calls']):>9} {opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " f"{s.get('infra_err', 0):>5} " - f"{_fmt(s['med_result_tokens'], 0):>8} {_fmt(s['p95_result_tokens'], 0):>8} " + f"{_fmt_result_tokens(s['med_result_tokens'], task_token_mode):>9} " + f"{_fmt_result_tokens(s['p95_result_tokens'], task_token_mode):>9} " f"{_fmt(s['med_cum_input'], 0):>10}" ) diff --git a/evals/run.py b/evals/run.py index c6146421..2ac1c4ba 100644 --- a/evals/run.py +++ b/evals/run.py @@ -381,6 +381,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: choices=("anthropic", "openai"), help="Model API provider for --driver api/sdk (default: anthropic).", ) + p.add_argument( + "--record-result-payloads", + action="store_true", + help=( + "CLI drivers only: record serialized tool-result text for tokenizer counting " + "(off by default; sidecars may contain live workspace data)" + ), + ) p.add_argument("--out", type=str, default=None, help="JSONL output path") p.add_argument( "--resume", @@ -478,7 +486,6 @@ async def run_agent_task_via_driver( optimal=optimal, alternate=alternate, classify=classify_call, - skip_result_tokens=agent_run.result_tokens_estimated is None, ) @@ -544,6 +551,7 @@ async def run_live( server_cmd: list[str] | None = None, server_env: dict[str, str] | None = None, resume: bool = False, + record_result_payloads: bool = False, ) -> int: surface = (surface or "full").strip().lower() external = server_cmd is not None @@ -608,6 +616,8 @@ async def run_live( driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) if driver_name == "codex-cli": driver_kwargs["allow_live"] = True + if not is_api_driver: + driver_kwargs["record_result_payloads"] = record_result_payloads # --server-cmd must reach every driver; otherwise we # silently benchmark the wrong server. if server_cmd is not None: @@ -968,6 +978,7 @@ def main(argv: list[str] | None = None) -> int: server_cmd=server_cmd, server_env=server_env or None, resume=bool(args.resume), + record_result_payloads=bool(args.record_result_payloads), ) ) diff --git a/evals/token_counting.py b/evals/token_counting.py new file mode 100644 index 00000000..cc20dada --- /dev/null +++ b/evals/token_counting.py @@ -0,0 +1,75 @@ +"""Shared tool-result token sizing for eval drivers and analysis.""" + +from __future__ import annotations + +from dataclasses import dataclass + +TOKEN_ESTIMATE_METHOD = "chars_div_4" +TOKENIZER_ENCODING = "cl100k_base" + + +@dataclass(frozen=True) +class ResultTokenCount: + """A tool-result token count and how it was obtained.""" + + value: int + estimated: bool + method: str + tokenizer_failed: bool = False + + +def estimate_result_tokens(result_chars: int) -> int: + """Deterministically estimate tokens from a recorded character count.""" + chars = max(0, int(result_chars)) + if chars == 0: + return 0 + return max(1, (chars + 3) // 4) + + +def count_result_text_tokens(text: str) -> ResultTokenCount: + """Count serialized result text with tiktoken, or identify an estimate. + + The optional import stays here, in the harness analysis process. The stdlib- + only recording proxy never imports this module. + """ + try: + import tiktoken + except ImportError: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + ) + except Exception: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + tokenizer_failed=True, + ) + + try: + encoding = tiktoken.get_encoding(TOKENIZER_ENCODING) + encode_ordinary = getattr(encoding, "encode_ordinary", None) + tokens = encode_ordinary(text) if callable(encode_ordinary) else encoding.encode(text) + return ResultTokenCount( + len(tokens), + estimated=False, + method=f"tiktoken:{TOKENIZER_ENCODING}", + ) + except Exception: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + tokenizer_failed=True, + ) + + +__all__ = [ + "TOKEN_ESTIMATE_METHOD", + "TOKENIZER_ENCODING", + "ResultTokenCount", + "count_result_text_tokens", + "estimate_result_tokens", +] diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py index c24784b9..93790475 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/test_evals_api_driver.py @@ -18,6 +18,7 @@ ToolSpec, Turn, ) +from evals.token_counting import estimate_result_tokens class FakeBackend: @@ -144,6 +145,10 @@ def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): {"in": 30, "out": 6, "cache_read": 9, "cache_write": 0}, ] assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] + assert [call["result_tokens"] for call in run.calls] == [ + estimate_result_tokens(len("first result")), + estimate_result_tokens(len("second")), + ] assert [call["is_error"] for call in run.calls] == [False, True] assert run.result_tokens_estimated is True assert run.token_count_failures == 0 @@ -297,7 +302,6 @@ def test_api_driver_maps_every_legacy_row_field(): classify=lambda tool, optimal, alternate: ( "optimal" if tool in optimal else "alternate" if tool in alternate else "out_of_set" ), - skip_result_tokens=False, ) required = { @@ -327,7 +331,7 @@ def test_api_driver_maps_every_legacy_row_field(): "is_error", } <= row["calls"][0].keys() assert row["calls"][0]["result_chars"] == 5 - assert row["calls"][0]["result_tokens"] == 2 + assert row["calls"][0]["result_tokens"] == estimate_result_tokens(5) == 2 assert row["result_tokens_estimated"] is True assert row["provider"] == "fake" assert row["model"] == "fake-actual" diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index fe23eb62..8d11067e 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -33,6 +33,7 @@ write_claude_mcp_config, ) from evals.run import classify_call, parse_args, stdio_server_env +from evals.token_counting import estimate_result_tokens # --------------------------------------------------------------------------- # Fixtures (constructed — never captured from live CLIs) @@ -700,7 +701,6 @@ def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): optimal={"find_work_items"}, alternate={"get_work_item"}, classify=classify_call, - skip_result_tokens=True, ) assert out["num_calls"] == 1 assert out["out_of_set_calls"] == 0 @@ -712,7 +712,10 @@ def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): assert out["cum_input_tokens_reason"] assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 assert out["usage_per_iteration"] == [] - assert out["result_tokens_skipped_reason"] + assert out["calls"][0]["result_tokens"] == 0 + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + assert "result_tokens_skipped_reason" not in out def test_agent_run_hit_max_maps_to_hit_max_iterations(): @@ -755,12 +758,88 @@ def test_agent_run_to_harness_dict_does_not_guess_usage_total(): optimal=set(), alternate=set(), classify=classify_call, - skip_result_tokens=True, ) assert out["usage"] == run.usage assert out["usage_total"] is None +def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): + class FakeEncoding: + def encode(self, text): + assert text == "serialized workspace result" + return [10, 20, 30] + + class FakeTiktoken: + @staticmethod + def get_encoding(name): + assert name == "cl100k_base" + return FakeEncoding() + + monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len("serialized workspace result"), + "result_text": "serialized workspace result", + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) + + assert out["calls"][0]["result_tokens"] == 3 + assert out["calls"][0]["result_tokens_estimated"] is False + assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" + assert out["result_tokens_estimated"] is False + assert out["result_tokens_mode"] == "measured" + assert "result_text" not in out["calls"][0] + + +def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): + monkeypatch.setitem(sys.modules, "tiktoken", None) + text = "payload without a tokenizer" + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len(text), + "result_text": text, + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) + + assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + + # --------------------------------------------------------------------------- # Plumbing # --------------------------------------------------------------------------- @@ -783,6 +862,9 @@ def test_parse_args_accepts_driver(): b = parse_args(["--dry-run"]) assert b.driver == "api" assert b.provider == "anthropic" + assert b.record_result_payloads is False + c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) + assert c.record_result_payloads is True def test_stdio_env_still_works_for_cli_drivers(monkeypatch): diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index 3fabfde2..c720c0b0 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -40,6 +40,7 @@ ) from evals.proxy import main as proxy_main from evals.run import resolve_model_for_driver +from evals.token_counting import estimate_result_tokens REPO = Path(__file__).resolve().parent.parent @@ -261,13 +262,69 @@ def test_sidecar_recorder_unit(tmp_path: Path): rec.on_server_message({"jsonrpc": "2.0", "id": 9, "result": {"content": [], "isError": False}}) rec.write_meta() calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") + raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] + raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") assert len(calls) == 1 assert calls[0]["tool"] == "t" assert calls[0]["args"] == {"a": 1} assert calls[0]["origin"] == "plane" + assert "result_text" not in calls[0] + assert "result_text" not in raw_call assert rec.finalized is True +def test_sidecar_result_payload_round_trips_only_when_enabled(tmp_path: Path): + path = tmp_path / "payload.jsonl" + rec = SidecarRecorder(path, record_result_payloads=True) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "find_work_items", "arguments": {}}, + } + ) + result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} + rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) + rec.write_meta() + + expected_text = json.dumps(result, default=str, ensure_ascii=False) + raw_call = next( + row + for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) + if row.get("row_type") != "proxy_meta" + ) + assert raw_call["result_text"] == expected_text + calls = load_proxy_sidecar_calls(path) + assert calls[0]["result_text"] == expected_text + assert calls[0]["result_chars"] == len(expected_text) + + +def test_old_payload_free_sidecar_still_parses(tmp_path: Path): + path = tmp_path / "old.jsonl" + path.write_text( + json.dumps( + { + "tool": "legacy", + "args": {}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n" + + json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}) + + "\n", + encoding="utf-8", + ) + + calls, status = load_proxy_sidecar(path) + assert status["state"] == "complete" + assert calls[0]["result_chars"] == 17 + assert "result_text" not in calls[0] + + def test_append_after_finalize_is_dropped(tmp_path: Path): """Once write_meta seals the sidecar, further row appends no-op (meta stays last).""" rec = SidecarRecorder(tmp_path / "fin.jsonl") @@ -507,6 +564,9 @@ def test_agent_run_to_harness_propagates_proxy_fields(): ) assert d["calls"][0]["is_error"] is True assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) + assert d["calls"][0]["result_tokens_estimated"] is True + assert d["result_tokens_estimated"] is True assert d["calls"][0]["duration_ms"] == 42 assert d["errored_calls"] == 1 @@ -611,6 +671,14 @@ def test_proxy_wrap_server_command(): assert out[5] == "--" assert out[6:] == ["python", "-m", "plane_mcp", "stdio"] + with_payloads = proxy_wrap_server_command( + ["server"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="python", + record_result_payloads=True, + ) + assert with_payloads[5:7] == ["--record-result-payloads", "--"] + def test_proxy_main_requires_command(): with pytest.raises(SystemExit): @@ -937,6 +1005,7 @@ def fake_run(cmd, **kwargs): kwargs = { "runner": make_fake(Driver, seen), "use_proxy": True, + "record_result_payloads": True, "python_bin": sys.executable, "server_command": ["/ext/bin/foreign-mcp", "stdio", "--v2"], } @@ -953,6 +1022,7 @@ def fake_run(cmd, **kwargs): ) blob = json.dumps(seen) assert "foreign-mcp" in blob or "foreign-mcp" in seen.get("cmd_joined", "") + assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index b3eb860a..1247fc6a 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -142,6 +142,57 @@ def test_summarize_aggregate_wilson_and_call_variance(): assert 0.0 <= meta["aggregate_wilson_lo"] <= meta["aggregate_wilson_hi"] <= 1.0 +def test_report_marks_entirely_estimated_result_token_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + } + ] + summary = summarize(rows) + assert summary["_meta"]["result_tokens_mode"] == "estimated" + assert summary["R1"]["result_tokens_mode"] == "estimated" + + report_mod.print_table(summary, "estimated") + output = capsys.readouterr().out + assert "entirely estimated" in output + assert "med_rtok~" in output + assert "~12" in output + + +def test_report_marks_mixed_measured_and_estimated_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], + "result_tokens_estimated": False, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + }, + ] + summary = summarize(rows) + assert summary["_meta"]["result_tokens_mode"] == "mixed" + assert summary["R1"]["result_tokens_mode"] == "mixed" + + report_mod.print_table(summary, "mixed") + output = capsys.readouterr().out + assert "mixed measured and estimated" in output + assert "med_rtok*" in output + + # --------------------------------------------------------------------------- # A/B compare # --------------------------------------------------------------------------- From c735b1c89924fffc1b10baca81cab6299a7ca601 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 01:52:26 +0530 Subject: [PATCH 05/93] Split the task catalog into one module per task class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tasks.py was 2798 lines: the catalog, 34 verifiers, answer matchers, API lookup helpers and prompt binding. Adding a single task meant opening all of it. Tasks now live with their own verifiers, grouped by class — read, write, schema, cross, debias — because those are edited together, so splitting data from behaviour would only mean two files open for every change. Shared machinery moves to common.py, and the package takes over the evals.tasks module path so every existing import is untouched. Task order is load-bearing: battery_fingerprint hashes the catalog and every result file we have carries that hash, so a reordering would silently invalidate comparisons against past runs rather than fail. The fingerprint is unchanged (6425dcc64404, 34 tasks) and a test now pins the id order. Co-Authored-By: Claude Fable 5 --- evals/tasks.py | 2798 ----------------------------------- evals/tasks/__init__.py | 329 ++++ evals/tasks/common.py | 272 ++++ evals/tasks/cross.py | 188 +++ evals/tasks/debias.py | 717 +++++++++ evals/tasks/read.py | 395 +++++ evals/tasks/schema.py | 540 +++++++ evals/tasks/write.py | 771 ++++++++++ tests/test_evals_catalog.py | 50 +- 9 files changed, 3261 insertions(+), 2799 deletions(-) delete mode 100644 evals/tasks.py create mode 100644 evals/tasks/__init__.py create mode 100644 evals/tasks/common.py create mode 100644 evals/tasks/cross.py create mode 100644 evals/tasks/debias.py create mode 100644 evals/tasks/read.py create mode 100644 evals/tasks/schema.py create mode 100644 evals/tasks/write.py diff --git a/evals/tasks.py b/evals/tasks.py deleted file mode 100644 index 43e5ad36..00000000 --- a/evals/tasks.py +++ /dev/null @@ -1,2798 +0,0 @@ -"""Task definitions (plain dicts) and verifier functions for the eval harness.""" - -from __future__ import annotations - -import hashlib -import json -import re -import string -from typing import Any - -from plane.errors.errors import HttpError -from plane.models.enums import PropertyType -from plane.models.query_params import RetrieveQueryParams, WorkItemQueryParams - -from evals.seed import ( - CUSTOMER_NAME, - CUSTOMER_REQUEST_NAME, - CYCLE_CURRENT, - CYCLE_PAST, - DEBIAS_CUSTOMER_PROP_DISPLAY, - DEBIAS_RELEASE_TAG_VERSION, - INTAKE_BILLING_TITLE, - INTAKE_SPAM_TITLE, - MODULE_COMPLETED_TITLES, - MODULE_NAME, - R1_TITLE, - R5_COMMENT_PHRASES, - R5_TITLE, - RELEASE_CHANGELOG_TEXT, - RELEASE_NAME, - W2_TITLE, - W3_TITLE, - W7_SOURCE_TITLE, - W7_TARGET_TITLE, - W7_URL, - W8_TITLE, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -class TaskSkipped(Exception): - """Verifier signals that this task-rep should be recorded as skipped, not failed.""" - - def __init__(self, reason: str) -> None: - super().__init__(reason) - self.reason = reason - - -class PromptBindError(RuntimeError): - """Live prompt could not bind required seed IDs (classified as infra_seed).""" - - -def format_task_prompt( - task: dict[str, Any], - ctx: dict[str, Any] | None = None, - *, - strict: bool = False, -) -> str: - """Render a task prompt with seed-bound placeholders. - - Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand - the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an - optional ``prompt_bind(ctx) -> dict`` callable on the task dict. - - When ``strict=True`` (live runs), empty-string values or binder exceptions - raise ``PromptBindError`` so the harness records ``infra_seed`` rather than - sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and - fills missing keys with explicit ```` markers. - """ - tpl = str(task.get("prompt") or "") - fields: dict[str, Any] = { - "project": (ctx or {}).get("project_name") or "EVAL deadbeef", - } - binder = task.get("prompt_bind") - if callable(binder) and ctx is not None: - try: - extra = binder(ctx) or {} - except Exception as exc: - if strict: - raise PromptBindError( - f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" - ) from exc - extra = {} - if isinstance(extra, dict): - for key, val in extra.items(): - if val is None: - if strict: - raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") - continue - text = str(val).strip() - if not text: - if strict: - raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") - continue - fields[key] = text - # Collect required placeholders from the template. - required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] - for name in required: - if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): - continue - if strict: - raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") - fields.setdefault(name, f"<{name}>") - return tpl.format(**fields) - - -def _word_boundary(value: str) -> re.Pattern[str]: - """Compile a case-insensitive word-boundary match for an exact seeded value.""" - return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) - - -def _reports_exact_int(text: str, n: int) -> bool: - """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" - return bool(_word_boundary(str(int(n))).search(text or "")) - - -def _whole_answer_int(text: str) -> int | None: - """If the answer (or its last non-empty line) is exactly an integer, return it. - - Letters must not appear — only surrounding whitespace/punctuation is ignored — - so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading - minus** attached to the number is preserved (``-3`` → -3, not 3). - """ - - def _as_int(s: str) -> int | None: - # Collapse whitespace; then the whole string must be optional sign + digits - # with only non-word punctuation wrappers (prefix must not eat the sign). - compact = re.sub(r"\s+", "", s or "") - m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) - if m: - return int(m.group(1)) - return None - - blob = text or "" - v = _as_int(blob) - if v is not None: - return v - lines = [ln for ln in blob.splitlines() if ln.strip()] - if lines: - return _as_int(lines[-1]) - return None - - -def reports_contract_int(text: str, truth: int) -> bool: - """True when final text reports ``truth`` via the explicit ``count: N`` contract. - - 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding - whitespace allowed). Use the **last** match; require signed equality with - ``truth``. - 2. Fallback: whole-answer / last-line bare integer (:func:`_whole_answer_int`). - 3. No match at all → False (ignoring an explicit format instruction is a fail). - """ - last: int | None = None - for line in (text or "").splitlines(): - m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) - if m: - last = int(m.group(1)) - if last is not None: - return last == int(truth) - whole = _whole_answer_int(text) - if whole is not None: - return whole == int(truth) - return False - - -def _as_id(obj: Any) -> str | None: - if obj is None: - return None - if isinstance(obj, str): - return obj - return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) - - -def _ids(items: Any) -> set[str]: - out: set[str] = set() - for item in items or []: - i = _as_id(item) - if i: - out.add(str(i)) - return out - - -def _find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: - """Return all work items with exact name, newest first (by created_at).""" - matches: list[Any] = [] - cursor = None - while True: - params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) - page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) - for item in page.results or []: - if (item.name or "").strip() == name: - matches.append(item) - if not page.next_page_results: - break - cursor = page.next_cursor - - def _created_key(item: Any) -> str: - return str(getattr(item, "created_at", None) or "") - - matches.sort(key=_created_key, reverse=True) - return matches - - -def _find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: - """Locate a work item by exact name; when duplicates exist, prefer the newest.""" - matches = _find_items_by_name(plane, workspace_slug, project_id, name) - return matches[0] if matches else None - - -def _state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: - """Resolve a state UUID or expanded object to its display name.""" - if state_ref is None: - return None - if hasattr(state_ref, "name") and state_ref.name: - return str(state_ref.name) - if isinstance(state_ref, dict) and state_ref.get("name"): - return str(state_ref["name"]) - state_id = _as_id(state_ref) - if not state_id: - return None - try: - state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) - return state.name - except HttpError as exc: - if exc.status_code not in (404, 405): - raise - # Fall back to listing states and matching by id. - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - results = page.results if hasattr(page, "results") else page - for s in results or []: - if str(s.id) == str(state_id): - return s.name - return None - - -def _state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: - if state_ref is None: - return None - if hasattr(state_ref, "group") and state_ref.group: - return str(state_ref.group) - if isinstance(state_ref, dict) and state_ref.get("group"): - return str(state_ref["group"]) - state_id = _as_id(state_ref) - if not state_id: - return None - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - for s in page.results or []: - if str(s.id) == str(state_id): - return getattr(s, "group", None) - return None - - -def _is_not_found(exc: BaseException) -> bool: - return isinstance(exc, HttpError) and exc.status_code in (404, 405) - - -def _final_text(run: dict[str, Any]) -> str: - return run.get("final_text") or "" - - -def _count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: - """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} - n = 0 - cursor = None - while True: - params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) - resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) - for item in resp.results or []: - if (getattr(item, "priority", None) or "").lower() != "urgent": - continue - sid = _as_id(item.state) - if sid and str(sid) in closed_ids: - continue - n += 1 - if not resp.next_page_results: - break - cursor = resp.next_cursor - return n - - -# --------------------------------------------------------------------------- -# Verifiers — async (plane, ctx, run) -> (bool, note) -# --------------------------------------------------------------------------- - - -async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R1: final text must name the target item's state and no other seeded state. - - Matching rule: word-boundary, case-insensitive regex on the exact state name - resolved from the API at verify time (never hardcoded). Additionally fail if - any *other* project state name also matches (blocks guessing/list_states echo). - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - title = R1_TITLE - item = _find_item_by_name(plane, workspace_slug, project_id, title) - if item is None: - return False, f"seeded item {title!r} not found" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - expected = _state_name(plane, workspace_slug, project_id, detail.state) - if not expected: - # Prefer the seeded name when API is sparse. - expected = ctx.get("r1_state_name") - if not expected: - return False, "could not resolve expected state name from API" - - final_text = _final_text(run) - if not _word_boundary(expected).search(final_text): - return False, f"final text missing state name {expected!r}" - - other_states = [n for n in (ctx.get("state_names") or []) if n and n.casefold() != expected.casefold()] - collisions = [n for n in other_states if _word_boundary(n).search(final_text)] - if collisions: - return ( - False, - f"final text names other state(s) {collisions!r} besides expected {expected!r}", - ) - return True, f"final text names only state {expected!r}" - - -async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R2: final text must contain the exact urgent-open count (word-boundary).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - expected = _count_open_urgent(plane, workspace_slug, project_id) - final_text = _final_text(run) - # Word-boundary on the decimal form of the count (blocks "4" matching "24"). - if not _word_boundary(str(expected)).search(final_text): - return False, f"final text missing urgent-open count {expected}" - return True, f"final text names count {expected}" - - -async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R3: final text must include each seeded assigned-to-me / due-this-week title.""" - titles = list(ctx.get("r3_due_titles") or []) - if not titles: - return False, "no R3 due titles in seed ctx" - final_text = _final_text(run) - missing = [t for t in titles if not _word_boundary(t).search(final_text)] - if missing: - return False, f"final text missing title(s) {missing!r}" - return True, f"final text names {len(titles)} due-this-week assigned items" - - -async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R4: final text must mention the active cycle name and the overdue item title.""" - final_text = _final_text(run) - notes: list[str] = [] - ok = True - if not _word_boundary(CYCLE_CURRENT).search(final_text): - ok = False - notes.append(f"missing active cycle {CYCLE_CURRENT!r}") - else: - notes.append(f"names {CYCLE_CURRENT}") - overdue = ctx.get("r4_overdue_title") - if overdue: - if not _word_boundary(overdue).search(final_text): - # Soft: also accept "overdue" keyword + any active item title. - if "overdue" not in final_text.casefold(): - ok = False - notes.append(f"missing overdue title {overdue!r}") - else: - notes.append("mentions overdue (title not exact)") - else: - notes.append(f"names overdue {overdue!r}") - return ok, "; ".join(notes) - - -async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R5: final text must include seeded comment phrases (word-boundary).""" - phrases = list(ctx.get("r5_comment_phrases") or R5_COMMENT_PHRASES) - final_text = _final_text(run) - missing = [p for p in phrases if not _word_boundary(p).search(final_text)] - if missing: - return False, f"final text missing comment phrase(s) {missing!r}" - return True, f"final text names {len(phrases)} discussion phrases" - - -async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R6: final text must name the project that has more open bugs (resolved at verify).""" - expected = ctx.get("r6_more_bugs_project") or ctx.get("second_project_name") - if not expected: - return False, "second project name missing from seed ctx" - final_text = _final_text(run) - # Match the full project name or the distinctive " B" suffix run8 form. - if _word_boundary(expected).search(final_text): - return True, f"final text names project with more bugs {expected!r}" - # Allow matching just the identifier-ish trailing token (e.g. run8 + B). - run8 = ctx.get("run8") or "" - alt = f"EVAL {run8} B" - if _word_boundary(alt).search(final_text) or (run8 and run8 in final_text and " B" in final_text): - return True, f"final text names second project ({alt})" - return False, f"final text missing project with more bugs {expected!r}" - - -async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W1: assert end-state via Plane API (title, priority, assignee, auth label).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - title = "Login page 500s on empty password" - matches = _find_items_by_name(plane, workspace_slug, project_id, title) - if not matches: - return False, f"work item {title!r} not found" - item = matches[0] # newest first - notes: list[str] = [] - if len(matches) > 1: - notes.append(f"warning: {len(matches)} items with title (verifying newest)") - - detail = plane.work_items.retrieve( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=item.id, - params=RetrieveQueryParams(expand="assignees,labels"), - ) - - ok = True - - priority = (detail.priority or "").lower() if detail.priority else "" - if priority != "urgent": - ok = False - notes.append(f"priority={priority!r} (want urgent)") - else: - notes.append("priority=urgent") - - me = plane.users.get_me() - me_id = str(me.id) - assignee_ids = _ids(detail.assignees) - if me_id not in assignee_ids: - ok = False - notes.append(f"assignees={sorted(assignee_ids)} missing me={me_id}") - else: - notes.append("assigned to me") - - auth_label_id = (ctx.get("labels") or {}).get("auth") - label_ids = _ids(detail.labels) - if not auth_label_id: - ok = False - notes.append("auth label id missing from seed ctx") - elif str(auth_label_id) not in label_ids: - ok = False - notes.append(f"labels={sorted(label_ids)} missing auth={auth_label_id}") - else: - notes.append("auth label attached") - - return ok, "; ".join(notes) - - -async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W2: target item is in a completed-group state (prefer name Done).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - item = _find_item_by_name(plane, workspace_slug, project_id, W2_TITLE) - if item is None: - return False, f"item {W2_TITLE!r} not found" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - name = _state_name(plane, workspace_slug, project_id, detail.state) - group = _state_group(plane, workspace_slug, project_id, detail.state) - if group == "completed" or (name and name.casefold() == "done"): - return True, f"state={name!r} group={group!r}" - return False, f"state={name!r} group={group!r} (want completed/Done)" - - -async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W3: target item has a comment containing the prompt phrase 'contrast tokens'.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - item = _find_item_by_name(plane, workspace_slug, project_id, W3_TITLE) - if item is None: - return False, f"item {W3_TITLE!r} not found" - resp = plane.work_items.comments.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=item.id, - ) - results = list(resp.results if hasattr(resp, "results") else resp or []) - if not results: - return False, "no comments on target item" - phrase = "contrast tokens" - pat = _word_boundary(phrase) - for c in results: - html = getattr(c, "comment_html", None) or "" - stripped = getattr(c, "comment_stripped", None) or "" - # Some APIs expose plain text under comment_stripped; fall back to html. - blob = f"{stripped}\n{html}" - if pat.search(blob): - return True, f"comment matches {phrase!r}" - return False, f"no comment contains {phrase!r} ({len(results)} comment(s))" - - -async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W4: the seeded triage label id is now named needs-triage. - - Authoritative path: retrieve ctx['labels']['triage'] by id. Name-scan is - only a fallback when the seed id is missing from ctx. - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - triage_id = (ctx.get("labels") or {}).get("triage") - if triage_id: - try: - lb = plane.labels.retrieve(workspace_slug=workspace_slug, project_id=project_id, label_id=triage_id) - name = (lb.name or "").strip().casefold() - if name in ("needs-triage", "needs triage"): - return True, f"label id {triage_id} now named {lb.name!r}" - return False, f"label id {triage_id} still named {lb.name!r}" - except HttpError as exc: - if not _is_not_found(exc): - raise - return False, f"seeded triage label id {triage_id} not found (deleted?)" - - # Fallback only when seed id is absent from ctx. - page = plane.labels.list(workspace_slug=workspace_slug, project_id=project_id) - names = {(lb.name or "").strip().casefold(): (lb.name or "").strip() for lb in (page.results or [])} - if "needs-triage" in names or "needs triage" in names: - if "triage" in names: - return False, "both triage and needs-triage still present" - return True, "label renamed to needs-triage (no seed id; name-scan fallback)" - return False, f"needs-triage not found; labels={sorted(names.values())}" - - -async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W5: all seeded module completed items are archived (not merely deleted).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - ids = [str(i) for i in (ctx.get("module_completed_ids") or [])] - if not ids: - # Fall back to titles. - for title in MODULE_COMPLETED_TITLES: - item = _find_item_by_name(plane, workspace_slug, project_id, title) - if item: - ids.append(str(item.id)) - if not ids: - return False, "no module completed item ids" - - not_archived: list[str] = [] - need_archive_list: list[str] = [] # 404 on retrieve — must appear in archived list - for wid in ids: - try: - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - except HttpError as exc: - if _is_not_found(exc): - # Deleted OR archived-as-404 — require confirmation via list_archived. - need_archive_list.append(str(wid)) - continue - raise - archived_at = getattr(detail, "archived_at", None) - if not archived_at: - not_archived.append(str(wid)) - - arch_ids: set[str] = set() - if need_archive_list or not_archived: - try: - arch = plane.work_items.list_archived( - workspace_slug=workspace_slug, - project_id=project_id, - params=WorkItemQueryParams(per_page=100), - ) - arch_ids = {str(i.id) for i in (arch.results or [])} - except Exception as exc: - if need_archive_list: - return False, f"list_archived failed while confirming 404 items: {exc}" - - # 404s only count as archived if present on the archived list (deletes fail). - for wid in need_archive_list: - if wid not in arch_ids: - not_archived.append(wid) - not_archived = [i for i in not_archived if i not in arch_ids] - - if not_archived: - return False, f"{len(not_archived)} module items not archived: {not_archived}" - return True, f"{len(ids)} module completed items archived" - - -async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W6: Sprint 12 closed by a real completion signal + unfinished items on Sprint 13. - - complete_cycle (SDK) sets end_date to *today* — a no-op agent leaves the seeded - past end_date unchanged, so requiring end_date==today (or archived_at set) is - non-vacuous. progress_snapshot non-null is also accepted when the API flips it. - """ - from datetime import date as _date - - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - past_id = ctx.get("cycle_past_id") or (ctx.get("cycles") or {}).get(CYCLE_PAST) - cur_id = ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) - if not past_id: - return False, "Sprint 12 id missing from seed" - notes: list[str] = [] - ok = True - past = plane.cycles.retrieve(workspace_slug=workspace_slug, project_id=project_id, cycle_id=past_id) - end = getattr(past, "end_date", None) - archived_at = getattr(past, "archived_at", None) - snapshot = getattr(past, "progress_snapshot", None) - today = _date.today().isoformat() - seed_end = ctx.get("cycle_past_seed_end_date") - - # Real close signals (any one suffices): - # 1) complete_cycle → end_date becomes today - # 2) manage_cycle_archive → archived_at set - # 3) progress_snapshot populated (Plane completion snapshot) - # end_date comes back as a timestamp ('2026-08-12T00:00:00Z'), so compare the - # date part — a whole-string match against today's date can never be true. - end_day = str(end or "")[:10] - closed = False - if archived_at: - closed = True - notes.append(f"Sprint 12 archived_at={archived_at}") - elif end_day == today: - closed = True - notes.append(f"Sprint 12 end_date={end} (complete_cycle today)") - elif snapshot not in (None, {}, []): - closed = True - notes.append("Sprint 12 progress_snapshot set") - if not closed: - ok = False - notes.append( - f"Sprint 12 not closed: end_date={end!r} seed_end={seed_end!r} " - f"archived_at={archived_at!r} snapshot={snapshot!r} " - f"(want end_date={today!r} or archived_at or progress_snapshot)" - ) - - unfinished = list(ctx.get("w6_unfinished_titles") or []) - if cur_id and unfinished: - try: - on13 = plane.cycles.list_work_items( - workspace_slug=workspace_slug, - project_id=project_id, - cycle_id=cur_id, - params=WorkItemQueryParams(per_page=100), - ) - names = {(i.name or "").strip() for i in (on13.results or [])} - missing = [t for t in unfinished if t not in names] - if missing: - ok = False - notes.append(f"unfinished not on Sprint 13: {missing}") - else: - notes.append(f"{len(unfinished)} unfinished on Sprint 13") - except Exception as exc: - notes.append(f"list Sprint 13 items failed: {exc}") - return ok, "; ".join(notes) - - -async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W7: source blocks target (dependency) AND reference URL link exists on source. - - Only dump['blocking'] ids count — a reverse blocked_by match must not pass. - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - src = _find_item_by_name(plane, workspace_slug, project_id, W7_SOURCE_TITLE) - tgt = _find_item_by_name(plane, workspace_slug, project_id, W7_TARGET_TITLE) - if not src or not tgt: - return False, "W7 source/target items not found" - notes: list[str] = [] - ok = True - - # Dependencies — require tgt in blocking specifically. - try: - deps = plane.work_items.dependencies.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=src.id, - ) - dump = deps.model_dump() if hasattr(deps, "model_dump") else (deps if isinstance(deps, dict) else {}) - blocking = dump.get("blocking") or [] - if isinstance(blocking, dict): - blocking = blocking.get("results") or list(blocking.values()) - blocking_ids = _ids(blocking) - # blocking may also be plain UUID strings - for b in blocking if isinstance(blocking, list) else []: - if isinstance(b, str): - blocking_ids.add(b) - if str(tgt.id) not in blocking_ids: - ok = False - blob_hit = str(tgt.id) in str(dump) - note = f"no blocking relation from source to {tgt.id}; blocking_ids={sorted(blocking_ids)}" - if blob_hit: - note += " (target id appears elsewhere in dump — wrong direction)" - notes.append(note) - else: - notes.append("blocking relation present") - except Exception as exc: - ok = False - notes.append(f"dependencies list failed: {exc}") - - # Links - try: - links = plane.work_items.links.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=src.id, - ) - rows = links.results if hasattr(links, "results") else links - urls = {(getattr(ln, "url", None) or "").strip() for ln in (rows or [])} - if W7_URL not in urls: - ok = False - notes.append(f"link {W7_URL!r} missing; have {sorted(urls)}") - else: - notes.append("reference URL present") - except Exception as exc: - ok = False - notes.append(f"links list failed: {exc}") - - return ok, "; ".join(notes) - - -async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W8: work log of exactly 120 minutes exists on the target item. - - Note: plane-sdk Create work log has no logged-date field — 'yesterday' in the - prompt cannot be asserted; only duration is verified. - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - item = _find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) - if item is None: - return False, f"item {W8_TITLE!r} not found" - logs = plane.work_items.work_logs.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=item.id, - ) - rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) - durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] - if 120 in durations: - return True, "work log duration=120 present" - return False, f"no 120-minute work log; durations={durations}" - - -async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W9 (extra): bulk priority change — the three non-R1 urgent titles are now high.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - # All urgent fixtures except we ask agent to set medium-priority batch targets. - # Prompt targets the three titles starting with Session/Inventory/Checkout (non-R1 urgent). - targets = [ - "Checkout times out on 3DS challenge", - "Session cookie not rotated after login", - "Inventory count goes negative under load", - ] - wrong: list[str] = [] - for title in targets: - item = _find_item_by_name(plane, workspace_slug, project_id, title) - if not item: - wrong.append(f"{title}: missing") - continue - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - pr = (detail.priority or "").lower() - if pr != "high": - wrong.append(f"{title}: priority={pr!r}") - if wrong: - return False, "; ".join(wrong) - return True, "3 items priority=high" - - -async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W10 (extra): project page named Eval Runbook exists.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - try: - resp = plane.pages.list_project_pages(workspace_slug=workspace_slug, project_id=project_id) - rows = resp.results if hasattr(resp, "results") else resp - except Exception as exc: - return False, f"list pages failed: {exc}" - names = {(getattr(p, "name", None) or "").strip() for p in (rows or [])} - if "Eval Runbook" not in names: - return False, f"page 'Eval Runbook' missing; have {sorted(names)}" - return True, "page Eval Runbook present" - - -async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """S1: Bug type has an OPTION property 'Severity' with Critical/Major/Minor. - - Only type-scoped property listing is accepted (no project/workspace fallbacks that - would pass an unattached Severity). Unexpected API errors propagate as harness errors. - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - bug_type_id = ( - (ctx.get("bug_type") or {}).get("id") if isinstance(ctx.get("bug_type"), dict) else ctx.get("bug_type") - ) - if not bug_type_id: - raise TaskSkipped("bug_type not seeded") - - try: - props = list( - plane.work_item_properties.list( - workspace_slug=workspace_slug, - project_id=project_id, - type_id=str(bug_type_id), - ) - or [] - ) - except HttpError as exc: - if _is_not_found(exc): - return False, "Severity property not found on Bug type (type-scoped list empty/404)" - raise - - severity = None - for p in props: - display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() - if display.lower() == "severity": - # Prefer an explicit type link when the API exposes it. - issue_type = getattr(p, "issue_type", None) - if issue_type is not None and str(issue_type) not in ("", str(bug_type_id)): - continue - severity = p - break - if severity is None: - return False, "Severity property not found on Bug type" - - prop_type = getattr(severity, "property_type", None) - prop_type_val = prop_type.value if isinstance(prop_type, PropertyType) else prop_type - if str(prop_type_val or "").upper() != PropertyType.OPTION.value: - return False, f"Severity property_type={prop_type_val!r} (want OPTION)" - - option_names = { - (getattr(o, "name", None) or (o.get("name") if isinstance(o, dict) else "") or "").strip() - for o in (getattr(severity, "options", None) or []) - } - if not option_names: - try: - opts = plane.work_item_properties.options.list( - workspace_slug=workspace_slug, - project_id=project_id, - property_id=severity.id, - ) - option_names = {(getattr(o, "name", None) or "").strip() for o in (opts or [])} - except HttpError as exc: - if not _is_not_found(exc): - raise - option_names = set() - - required = {"critical", "major", "minor"} - have = {n.casefold() for n in option_names if n} - missing = required - have - if missing: - return False, f"Severity options missing {sorted(missing)}; have {sorted(option_names)}" - return True, "Severity OPTION with Critical/Major/Minor present on Bug type" - - -async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """S2: Fibonacci estimate scale exists and target item estimate_point is 5.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - notes: list[str] = [] - ok = True - - # Resolve active estimate + points. - try: - est = plane.estimates.retrieve(workspace_slug=workspace_slug, project_id=project_id) - except Exception as exc: - return False, f"no project estimate: {exc}" - est_id = getattr(est, "id", None) or _as_id(est) - points = plane.estimates.list_points(workspace_slug=workspace_slug, project_id=project_id, estimate_id=est_id) - point_rows = points if isinstance(points, list) else (points.results if hasattr(points, "results") else points) - values = {(getattr(p, "value", None) or "").strip() for p in (point_rows or [])} - fib_like = {"1", "2", "3", "5", "8"} - if not fib_like.issubset(values): - ok = False - notes.append(f"estimate points missing fib subset; have {sorted(values)}") - else: - notes.append("fibonacci points present") - - five = next((p for p in (point_rows or []) if (getattr(p, "value", None) or "").strip() == "5"), None) - item = _find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) - if item is None: - ok = False - notes.append(f"target item {W8_TITLE!r} missing") - else: - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - ep = getattr(detail, "estimate_point", None) - ep_id = _as_id(ep) if not isinstance(ep, (int, float)) else None - # estimate_point may be expanded object or UUID. - if five is not None and ep_id and str(ep_id) == str(five.id): - notes.append("item estimate_point=5") - elif ep is not None and str(getattr(ep, "value", ep)) in ("5", "5.0"): - notes.append("item estimate value=5") - else: - ok = False - notes.append(f"item estimate_point={ep!r} (want 5)") - return ok, "; ".join(notes) - - -async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """S3: Incident type exists with a required TEXT property. - - Workspace-owned types: probe get_features.is_work_item_types_enabled at verify - time (S3 needs is empty, so seed never sets bug_type_workspace_level). - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - # Find Incident type (project list first). - types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) - incident = next((t for t in types if (t.name or "").strip().casefold() == "incident"), None) - if incident is None: - # Probe workspace feature at verify time — do not rely on seed ctx flags. - workspace_owns = False - try: - features = plane.workspaces.get_features(workspace_slug=workspace_slug) - dump = features.model_dump() if hasattr(features, "model_dump") else {} - workspace_owns = bool(dump.get("is_work_item_types_enabled")) - except Exception: - workspace_owns = False - if workspace_owns: - try: - wtypes = list(plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []) - incident = next((t for t in wtypes if (t.name or "").strip().casefold() == "incident"), None) - except Exception: - pass - if incident is None: - return False, "Incident work item type not found" - - try: - props = list( - plane.work_item_properties.list( - workspace_slug=workspace_slug, - project_id=project_id, - type_id=str(incident.id), - ) - or [] - ) - except HttpError as exc: - if _is_not_found(exc): - return False, "no properties on Incident type" - raise - - required_text = None - for p in props: - prop_type = getattr(p, "property_type", None) - prop_type_val = str(prop_type.value if isinstance(prop_type, PropertyType) else prop_type or "").upper() - is_required = bool(getattr(p, "is_required", False)) - # TEXT only — no fallback to other required types (OPTION etc.). - if is_required and prop_type_val in (PropertyType.TEXT.value, "TEXT", "STRING"): - required_text = p - break - if required_text is None: - return False, f"no required TEXT property on Incident; props={len(props)}" - display = getattr(required_text, "display_name", None) or getattr(required_text, "name", None) - return True, f"Incident type + required TEXT property {display!r}" - - -async def verify_s4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """S4: billing intake accepted (status=1), spam declined (status=-1).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - intake = ctx.get("intake") or {} - billing = intake.get("billing") or {} - spam = intake.get("spam") or {} - notes: list[str] = [] - ok = True - - def _status_of(issue_id: str | None, title: str) -> int | None: - if not issue_id: - return None - try: - row = plane.intake.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=issue_id) - return getattr(row, "status", None) - except Exception: - # Fall back to list + match title - try: - rows = plane.intake.list(workspace_slug=workspace_slug, project_id=project_id) - results = rows.results if hasattr(rows, "results") else rows - for r in results or []: - detail = getattr(r, "issue_detail", None) - name = getattr(detail, "name", None) if detail is not None else None - if name and name.strip() == title: - return getattr(r, "status", None) - except Exception: - return None - return None - - b_status = _status_of(billing.get("issue_id"), INTAKE_BILLING_TITLE) - s_status = _status_of(spam.get("issue_id"), INTAKE_SPAM_TITLE) - # accept=1, decline=-1 per IntakeWorkItemStatusEnum - if b_status != 1: - ok = False - notes.append(f"billing status={b_status!r} (want 1/accepted)") - else: - notes.append("billing accepted") - if s_status != -1: - ok = False - notes.append(f"spam status={s_status!r} (want -1/declined)") - else: - notes.append("spam declined") - return ok, "; ".join(notes) - - -async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """S5: project cycles + worklogs AND workspace customers enabled. - - Gates (plane-ee): - - project.cycle_view — cycles create/list - - project.is_time_tracking_enabled — worklogs - - WorkspaceFeature.is_customer_enabled (API field ``customers``) — customer create 403 - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - notes: list[str] = [] - ok = True - - proj = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) - cycle_view = bool(getattr(proj, "cycle_view", None)) - time_tracking = bool(getattr(proj, "is_time_tracking_enabled", None)) - if not cycle_view: - ok = False - notes.append(f"cycle_view={getattr(proj, 'cycle_view', None)!r} (want True)") - else: - notes.append("cycle_view=True") - if not time_tracking: - ok = False - notes.append(f"is_time_tracking_enabled={getattr(proj, 'is_time_tracking_enabled', None)!r} (want True)") - else: - notes.append("is_time_tracking_enabled=True") - - try: - feat = plane.projects.get_features(workspace_slug=workspace_slug, project_id=project_id) - dump = feat.model_dump() if hasattr(feat, "model_dump") else (feat if isinstance(feat, dict) else {}) - cycles_flag = dump.get("cycles") if isinstance(dump, dict) else getattr(feat, "cycles", None) - if not cycles_flag: - ok = False - notes.append(f"features.cycles={cycles_flag!r} (want True)") - else: - notes.append("features.cycles=True") - except Exception as exc: - ok = False - notes.append(f"project get_features failed: {exc}") - - # Workspace customers toggle (is_customer_enabled behind API field ``customers``). - try: - ws_feat = plane.workspaces.get_features(workspace_slug=workspace_slug) - ws_dump = ( - ws_feat.model_dump() if hasattr(ws_feat, "model_dump") else (ws_feat if isinstance(ws_feat, dict) else {}) - ) - customers_on = None - if isinstance(ws_dump, dict): - customers_on = ws_dump.get("customers") - if customers_on is None: - customers_on = ws_dump.get("is_customer_enabled") - if customers_on is None: - customers_on = getattr(ws_feat, "customers", None) - if customers_on is None: - customers_on = getattr(ws_feat, "is_customer_enabled", None) - if not customers_on: - ok = False - notes.append(f"workspace.customers={customers_on!r} (want True)") - else: - notes.append("workspace.customers=True") - except Exception as exc: - ok = False - notes.append(f"workspace get_features failed: {exc}") - - return ok, "; ".join(notes) - - -async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """C1: customer 'Acme Corp' has request 'SSO support' linked to the R1 work item. - - Anchors: exact customer name, exact request name, and the R1_TITLE work item id - resolved from the eval project at verify time (must be among linked ids). - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - notes: list[str] = [] - ok = True - - # Resolve the required link target first (seeded Payment webhook item). - r1 = _find_item_by_name(plane, workspace_slug, project_id, R1_TITLE) - if r1 is None: - return False, f"R1 item {R1_TITLE!r} not found in project" - - customers = plane.customers.list(workspace_slug=workspace_slug) - rows = customers.results if hasattr(customers, "results") else customers - # Exact name only — do not match arbitrary acme* customers. - acme = next((c for c in (rows or []) if (c.name or "").strip() == CUSTOMER_NAME), None) - if acme is None: - return False, f"customer {CUSTOMER_NAME!r} not found" - - # Track for teardown if agent-created - if not ctx.get("customer"): - ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": acme.id}) - - reqs = plane.customers.requests.list(workspace_slug=workspace_slug, customer_id=acme.id) - rrows = reqs.results if hasattr(reqs, "results") else reqs - sso = next( - (r for r in (rrows or []) if (r.name or "").strip() == CUSTOMER_REQUEST_NAME), - None, - ) - if sso is None: - ok = False - notes.append(f"request {CUSTOMER_REQUEST_NAME!r} missing") - else: - notes.append("SSO request present") - - # Require the R1 work item among customer-linked work items. - try: - wi = plane.customers.work_items.list(workspace_slug=workspace_slug, customer_id=acme.id) - wi_rows = list(wi.results if hasattr(wi, "results") else wi or []) - linked_ids = _ids(wi_rows) - # Plain string ids also count. - for row in wi_rows: - if isinstance(row, str): - linked_ids.add(row) - elif isinstance(row, dict) and row.get("id"): - linked_ids.add(str(row["id"])) - else: - # Customer work item wrappers may expose work_item / issue field. - for attr in ("work_item", "work_item_id", "issue", "issue_id"): - ref = getattr(row, attr, None) if not isinstance(row, dict) else row.get(attr) - rid = _as_id(ref) - if rid: - linked_ids.add(str(rid)) - if str(r1.id) not in linked_ids: - ok = False - notes.append(f"R1 item {r1.id} not linked; linked={sorted(linked_ids)}") - else: - notes.append(f"R1 item {r1.id} linked") - except Exception as exc: - ok = False - notes.append(f"list customer work items failed: {exc}") - - return ok, "; ".join(notes) - - -async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """C2: final text mentions release 1.2.0 and at least one seeded changelog phrase.""" - final_text = _final_text(run) - notes: list[str] = [] - ok = True - if not _word_boundary(RELEASE_NAME).search(final_text): - ok = False - notes.append(f"missing release name {RELEASE_NAME!r}") - else: - notes.append(f"names {RELEASE_NAME}") - changelog = ctx.get("release_changelog_text") or RELEASE_CHANGELOG_TEXT - # Match distinctive fragments from the seeded changelog. - fragments = ["OAuth login hardening", "webhook retry backoff"] - hit = [f for f in fragments if _word_boundary(f).search(final_text)] - if not hit: - # Also accept substring of full changelog without word-boundary if short. - if changelog[:40].casefold() not in final_text.casefold(): - ok = False - notes.append("missing changelog content") - else: - notes.append("changelog substring present") - else: - notes.append(f"changelog phrases {hit}") - return ok, "; ".join(notes) - - -async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R7 (extra): final text names at least one legal next state for the R1 item. - - Resolves available completed/started/unstarted states at verify time and - requires a word-boundary hit on one of them (or explicit 'unrestricted'). - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - names = [(s.name or "").strip() for s in (page.results or []) if (s.name or "").strip()] - final_text = _final_text(run) - if "unrestricted" in final_text.casefold() or "any state" in final_text.casefold(): - return True, "agent reported unrestricted transitions" - hits = [n for n in names if _word_boundary(n).search(final_text)] - if not hits: - return False, f"final text names none of project states {names}" - return True, f"final text names state(s) {hits}" - - -# --------------------------------------------------------------------------- -# ID-in-hand (I*) + long-tail (L*) de-biasing verifiers -# --------------------------------------------------------------------------- - -# Stable titles used by ID-in-hand binders (seeded by needs={"items", ...}). -I1_TITLE = R1_TITLE # update priority by UUID -I2_TITLE = W2_TITLE # fetch by PROJ-N identifier -I3_TITLE = "Footer year still says 2024" # add to cycle by UUIDs (not on a cycle) -I4_TITLE = W3_TITLE # attach label by UUIDs -L1_TITLE = W8_TITLE # worklog + project summary -L2_TITLE = R5_TITLE # activities (seeded comments produce activity rows) -L5_TITLE = R1_TITLE # attachment listing -L3_TAG_VERSION = DEBIAS_RELEASE_TAG_VERSION -L4_PROP_DISPLAY = DEBIAS_CUSTOMER_PROP_DISPLAY -L4_PROP_VALUE = "Enterprise" - - -def _bind_item_uuid(title: str): - def _bind(ctx: dict[str, Any]) -> dict[str, str]: - wid = str((ctx.get("items") or {}).get(title) or "") - return {"work_item_id": wid} - - return _bind - - -def _bind_item_identifier(title: str): - def _bind(ctx: dict[str, Any]) -> dict[str, str]: - ident = str((ctx.get("item_identifiers") or {}).get(title) or "") - return {"work_item_identifier": ident} - - return _bind - - -def _bind_i3(ctx: dict[str, Any]) -> dict[str, str]: - wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") - cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") - return {"work_item_id": wid, "cycle_id": cycle_id} - - -def _bind_i4(ctx: dict[str, Any]) -> dict[str, str]: - wid = str((ctx.get("items") or {}).get(I4_TITLE) or "") - label_id = str((ctx.get("labels") or {}).get("perf") or "") - return {"work_item_id": wid, "label_id": label_id} - - -async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I1: seeded R1 item priority is high (updated by UUID, not name).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(I1_TITLE) - if not wid: - return False, f"seed item {I1_TITLE!r} missing" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - pr = (detail.priority or "").lower() if detail.priority else "" - if pr == "high": - return True, f"work_item {wid} priority=high" - return False, f"work_item {wid} priority={pr!r} (want high)" - - -async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I2: final text names the state of the identifier-target item.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(I2_TITLE) - if not wid: - return False, f"seed item {I2_TITLE!r} missing" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - name = _state_name(plane, workspace_slug, project_id, detail.state) - if not name: - return False, "target state name unresolved" - final_text = _final_text(run) - if _word_boundary(name).search(final_text): - return True, f"final text names state {name!r}" - return False, f"final text missing state {name!r}" - - -async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I3: work item UUID is on the target cycle UUID.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") - cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") - if not wid or not cycle_id: - return False, "seed work_item_id/cycle_id missing" - page = plane.cycles.list_work_items(workspace_slug=workspace_slug, project_id=project_id, cycle_id=cycle_id) - rows = page.results if hasattr(page, "results") else page - ids = {str(getattr(r, "id", None) or r) for r in (rows or [])} - # list may return issue wrappers with issue/id fields - for r in rows or []: - for attr in ("id", "issue", "work_item_id"): - v = getattr(r, attr, None) - if v is not None: - ids.add(str(v if not hasattr(v, "id") else v.id)) - if wid in ids: - return True, f"item {wid} on cycle {cycle_id}" - return False, f"item {wid} not on cycle {cycle_id}; have {sorted(ids)[:12]}" - - -async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I4: work item has the seeded perf label id attached.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(I4_TITLE) - label_id = (ctx.get("labels") or {}).get("perf") - if not wid or not label_id: - return False, "seed work_item_id/label_id missing" - detail = plane.work_items.retrieve( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=wid, - params=RetrieveQueryParams(expand="labels"), - ) - label_ids = _ids(detail.labels) - if str(label_id) in label_ids: - return True, f"label {label_id} on {wid}" - return False, f"labels={sorted(label_ids)} missing perf={label_id}" - - -async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I5: target item priority is low (updated by UUID).""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(I3_TITLE) - if not wid: - return False, f"seed item {I3_TITLE!r} missing" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - pr = (detail.priority or "").lower() if detail.priority else "" - if pr == "low": - return True, f"work_item {wid} priority=low" - return False, f"work_item {wid} priority={pr!r} (want low)" - - -def _l1_duration_reported(final_text: str) -> bool: - """Numeric duration only: whole-word 90 or 1.5 (not English 'ninety').""" - return bool(_word_boundary("90").search(final_text)) or bool(re.search(r"\b1\.5\b", final_text)) - - -def _l1_person_names_from_summary(sum_rows: Any) -> list[str]: - """Best-effort actor/assignee display strings from project worklog summary rows.""" - names: list[str] = [] - for row in sum_rows or []: - dump = row.model_dump() if hasattr(row, "model_dump") else {} - if not isinstance(dump, dict): - dump = {} - candidates: list[Any] = [] - for attr in ( - "actor", - "user", - "display_name", - "owned_by", - "created_by", - "assignee", - "email", - "first_name", - "last_name", - ): - v = getattr(row, attr, None) - if v is None and dump: - v = dump.get(attr) - if v is None: - continue - if hasattr(v, "display_name") or hasattr(v, "email"): - candidates.append( - getattr(v, "display_name", None) or getattr(v, "email", None) or getattr(v, "id", None) - ) - elif isinstance(v, dict): - candidates.append(v.get("display_name") or v.get("email") or v.get("id")) - else: - candidates.append(v) - for c in candidates: - s = str(c or "").strip() - if s and s not in names: - names.append(s) - return names - - -def _l1_summary_substance(final_text: str, *, title: str, sum_rows: Any) -> bool: - """Summary half of L1: item title, person from summary, or words summary/total. - - Deliberately does *not* accept bare 'logged' / 'worklog' — the prompt asks to - report the project worklog summary (who/what has time logged). - """ - low = final_text.casefold() - if "summary" in low or "total" in low: - return True - if title and _word_boundary(title).search(final_text): - return True - for person in _l1_person_names_from_summary(sum_rows): - if len(person) >= 2 and _word_boundary(person).search(final_text): - return True - return False - - -async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L1: 90-minute work log on the correct item AND final text reports duration + summary. - - Duration: numeric whole-word ``90`` or ``1.5`` only (not English 'ninety'). - Summary substance: item title, a person/assignee from the project summary, or - the words ``summary`` / ``total``. Bare "90 minutes of work" fails by design. - """ - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(L1_TITLE) - if not wid: - return False, f"seed item {L1_TITLE!r} missing" - # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). - logs = plane.work_items.work_logs.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) - durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] - if 90 not in durations: - return False, f"no 90-minute work log on target item {wid}; durations={durations}" - - sum_rows: list[Any] = [] - try: - summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) - raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) - sum_rows = list(raw or []) - except Exception: - # Summary fetch is optional for person names; duration + title/summary/total still work. - sum_rows = [] - - final_text = _final_text(run) - if not _l1_duration_reported(final_text): - return False, "final text missing logged duration (numeric 90 or 1.5)" - if not _l1_summary_substance(final_text, title=L1_TITLE, sum_rows=sum_rows): - return False, ( - "final text lacks worklog summary substance " - "(need item title, person from summary, or words 'summary'/'total')" - ) - return True, f"90m log on {wid} + final text reports duration and summary substance" - - -async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L2: target has activities; final text reports the count via ``count: N`` contract.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(L2_TITLE) - if not wid: - return False, f"seed item {L2_TITLE!r} missing" - try: - page = plane.work_items.activities.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - except Exception as exc: - return False, f"activities.list failed: {exc}" - rows = page.results if hasattr(page, "results") else page - n = len(list(rows or [])) - if n < 1: - return False, "no activities on target (seed comments should create some)" - final_text = _final_text(run) - if not reports_contract_int(final_text, n): - return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" - return True, f"final text reports activity count {n} via contract" - - -async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L5: final text reports the attachment count via ``count: N`` contract.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(L5_TITLE) - if not wid: - return False, f"seed item {L5_TITLE!r} missing" - try: - page = plane.work_items.attachments.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - except Exception as exc: - return False, f"attachments.list failed: {exc}" - rows = page.results if hasattr(page, "results") else page - n = len(list(rows or [])) - final_text = _final_text(run) - if not reports_contract_int(final_text, n): - return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" - return True, f"final text reports attachment count {n} via contract" - - -async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L3: workspace has a release tag with version eval-rc1.""" - workspace_slug = ctx["workspace_slug"] - try: - page = plane.releases.tags.list(workspace_slug=workspace_slug) - except Exception as exc: - return False, f"list release tags failed: {exc}" - rows = page.results if hasattr(page, "results") else page - versions = {(getattr(t, "version", None) or "").strip() for t in (rows or [])} - if L3_TAG_VERSION in versions: - # Track for teardown if id available. - for t in rows or []: - if (getattr(t, "version", None) or "").strip() == L3_TAG_VERSION: - tid = getattr(t, "id", None) - if tid: - objs = ctx.setdefault("workspace_objects", []) - if not any(o.get("kind") == "release_tag" and str(o.get("id")) == str(tid) for o in objs): - objs.append({"kind": "release_tag", "id": tid}) - break - return True, f"release tag {L3_TAG_VERSION!r} present" - return False, f"tag {L3_TAG_VERSION!r} missing; have {sorted(versions)}" - - -def _property_type_is_text(prop: Any) -> bool: - raw = getattr(prop, "property_type", None) - if raw is None: - raw = getattr(prop, "type", None) - if raw is None: - return False - if hasattr(raw, "value"): - raw = raw.value - if hasattr(raw, "name"): - # Enum member: PropertyType.TEXT - name = str(raw.name) - if name.upper() == "TEXT": - return True - s = str(raw).upper() - return s == "TEXT" or s.endswith(".TEXT") or s == "PROPERTYTYPE.TEXT" - - -async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L4: right customer has TEXT property 'Eval Industry' = Enterprise (exact name).""" - workspace_slug = ctx["workspace_slug"] - cust = ctx.get("customer") or {} - customer_id = cust.get("id") if isinstance(cust, dict) else cust - if not customer_id: - return False, "customer missing from seed" - try: - props = plane.customers.properties.list(workspace_slug=workspace_slug) - except Exception as exc: - return False, f"list customer properties failed: {exc}" - prop_rows = props.results if hasattr(props, "results") else props - target_prop: Any | None = None - for p in prop_rows or []: - # Exact display_name match only (case-insensitive full match) — not substring "Industry". - display = (getattr(p, "display_name", None) or "").strip() - if display.casefold() != L4_PROP_DISPLAY.casefold(): - continue - if not _property_type_is_text(p): - return False, ( - f"property {display!r} exists but property_type is not TEXT (got {getattr(p, 'property_type', None)!r})" - ) - target_prop = p - break - if target_prop is None: - return False, f"no TEXT customer property named exactly {L4_PROP_DISPLAY!r}" - pid = str(target_prop.id) - # Track for teardown. - objs = ctx.setdefault("workspace_objects", []) - if not any(o.get("kind") == "customer_property" and str(o.get("id")) == pid for o in objs): - objs.append({"kind": "customer_property", "id": pid}) - try: - values = plane.customers.property_values.list(workspace_slug=workspace_slug, customer_id=customer_id) - except Exception as exc: - return False, f"get property values failed: {exc}" - if not isinstance(values, dict): - return False, f"unexpected property_values shape: {type(values)}" - vals = values.get(pid) or values.get(str(pid)) or [] - flat = [str(v) for v in (vals if isinstance(vals, list) else [vals])] - if any(L4_PROP_VALUE.casefold() == v.casefold() for v in flat): - return True, f"customer {customer_id} property {pid} ({L4_PROP_DISPLAY})={L4_PROP_VALUE!r}" - return False, f"customer {customer_id} property {pid} values {flat} lack {L4_PROP_VALUE!r}" - - -# --------------------------------------------------------------------------- -# Task catalog (full DESIGN list + extras for uncovered v2 families) -# --------------------------------------------------------------------------- - -TASKS: list[dict[str, Any]] = [ - { - "id": "R1", - "tags": {"read", "tier1"}, - "prompt": ( - "In project {project}, what is the current state of the work item titled " - f"'{R1_TITLE}'? Answer with the state name." - ), - "optimal_calls": 1, - "optimal_tools": {"list_work_items"}, - "alternate_tools": { - "search_work_items", - "list_archived_work_items", - "count_work_items", - "retrieve_work_item", - "retrieve_work_item_by_identifier", - "list_projects", - "list_states", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "list_states", - "get_workspace_context", - "get_pql_reference", - }, - }, - }, - "needs": {"items"}, - "verify": verify_r1, - }, - { - "id": "R2", - "tags": {"read", "tier1"}, - "prompt": ( - "In project {project}, how many urgent open work items are there? Answer with the integer count only." - ), - "optimal_calls": 1, - "optimal_tools": {"count_work_items"}, - "alternate_tools": { - "list_work_items", - "search_work_items", - "list_projects", - "list_states", - "get_pql_reference", - }, - "surface_tools": { - "v2": { - # No count tool on v2 — find_work_items with priority/state filters is optimal. - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "list_states", - "get_workspace_context", - "get_pql_reference", - }, - }, - }, - "needs": {"items"}, - "verify": verify_r2, - }, - { - "id": "R3", - "tags": {"read", "tier1"}, - "prompt": ( - "In project {project}, list work items assigned to me that are due this week. Answer with their titles." - ), - "optimal_calls": 2, - "optimal_tools": {"get_me", "list_work_items"}, - "alternate_tools": { - "search_work_items", - "count_work_items", - "list_projects", - "get_workspace_members", - "get_pql_reference", - "retrieve_work_item", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_workspace_context", - "get_work_item", - "search_projects", - "get_pql_reference", - }, - }, - }, - "needs": {"items"}, - "verify": verify_r3, - }, - { - "id": "R4", - "tags": {"read", "tier1"}, - "prompt": ( - "In project {project}, what is in the active cycle, and is anything overdue? " - f"Name the cycle (expect '{CYCLE_CURRENT}') and any overdue item titles." - ), - "optimal_calls": 2, - "optimal_tools": {"list_cycles", "list_work_items"}, - "alternate_tools": { - "list_cycle_work_items", - "retrieve_cycle", - "search_work_items", - "list_projects", - "get_pql_reference", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "list_cycles", - "get_work_item", - "get_pql_reference", - "search_projects", - "get_workspace_context", - }, - }, - }, - "needs": {"items", "cycles"}, - "verify": verify_r4, - }, - { - "id": "R5", - "tags": {"read", "tier1"}, - "prompt": ( - f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " - "Include the key phrases from its comments." - ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_comments"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "retrieve_work_item_by_identifier", - "list_work_item_activities", - "list_projects", - }, - "surface_tools": { - "v2": { - # include= depth: single get_work_item with include=comments after resolve, - # or find + get with include. - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "get_work_item"}, - "alternate_tools": { - "search_projects", - "get_workspace_context", - "create_comment", - }, - }, - }, - "needs": {"items"}, - "verify": verify_r5, - }, - { - "id": "R6", - "tags": {"read", "tier1"}, - "prompt": ( - "Across the eval projects created for this run (main project {project} and its " - "sibling 'B' project), which project has more open Bug-typed work items? " - "Answer with the project name." - ), - "optimal_calls": 3, - "optimal_tools": {"list_projects", "list_work_items", "resolve_work_item_type"}, - "alternate_tools": { - "count_work_items", - "search_work_items", - "list_work_item_types", - "retrieve_project", - "get_pql_reference", - }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"search_projects", "find_work_items", "get_workspace_context"}, - "alternate_tools": { - "get_work_item", - "get_pql_reference", - "list_states", - }, - }, - # Type id resolution cleaner on v2-schema - "v2-schema": { - "optimal_calls": 3, - "optimal_tools": {"search_projects", "find_work_items", "resolve_work_item_type"}, - "alternate_tools": { - "list_work_item_types", - "get_workspace_context", - "get_work_item", - "get_pql_reference", - }, - }, - }, - "needs": {"items", "bug_type", "second_project"}, - "verify": verify_r6, - }, - { - "id": "W1", - "tags": {"write", "tier1"}, - "prompt": ( - "Create a work item in project {project}: title 'Login page 500s on empty " - "password', priority urgent, assign it to me, and add the 'auth' label." - ), - "optimal_calls": 4, - "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, - "alternate_tools": { - "search_work_items", - "list_states", - "retrieve_project", - "get_workspace_members", - "manage_work_item_assignee", - "manage_work_item_label", - "update_work_item", - "list_work_items", - }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"get_workspace_context", "create_work_item"}, - "alternate_tools": { - "search_projects", - "list_labels", - "find_work_items", - "get_work_item", - "update_work_item", - }, - }, - }, - "needs": {"labels"}, - "verify": verify_w1, - }, - { - "id": "W2", - "tags": {"write", "tier1"}, - "prompt": (f"In project {{project}}, move the work item titled '{W2_TITLE}' to the Done state."), - "optimal_calls": 3, - "optimal_tools": {"list_work_items", "list_states", "update_work_item"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "retrieve_state", - "list_projects", - }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "update_work_item"}, - "alternate_tools": { - "list_states", - "get_work_item", - "list_available_transitions", - "search_projects", - }, - }, - }, - "needs": {"items"}, - "verify": verify_w2, - }, - { - "id": "W3", - "tags": {"write", "tier1"}, - "prompt": ( - f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' " - "saying 'Reviewed contrast tokens — needs design pass'." - ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "create_work_item_comment"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "list_work_item_comments", - "list_projects", - }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "create_comment"}, - "alternate_tools": { - "get_work_item", - "modify_comment", - "search_projects", - }, - }, - }, - "needs": {"items"}, - "verify": verify_w3, - }, - { - "id": "W4", - "tags": {"write", "tier1"}, - "prompt": ("In project {project}, rename the label 'triage' to 'needs-triage'."), - "optimal_calls": 2, - "optimal_tools": {"list_labels", "update_label"}, - "alternate_tools": { - "retrieve_label", - "create_label", - "delete_label", - "list_projects", - }, - "surface_tools": { - "v2": { - # Default v2 has list_labels but no update_label (schema tier). - "unsupported": True, - "reason": ("W4 needs update_label which is only on the v2-schema surface — use --surface v2-schema"), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": {"list_labels", "update_label"}, - "alternate_tools": { - "create_label", - "delete_label", - "search_projects", - "get_workspace_context", - }, - }, - }, - "needs": {"labels"}, - "verify": verify_w4, - }, - { - "id": "W5", - "tags": {"write", "tier1"}, - "prompt": (f"In project {{project}}, archive all completed work items in the module '{MODULE_NAME}'."), - "optimal_calls": 5, # list_modules + list_module_work_items + 3× archive - "optimal_tools": { - "list_modules", - "list_module_work_items", - "manage_work_item_archive", - }, - "alternate_tools": { - "list_work_items", - "retrieve_module", - "list_projects", - "list_states", - }, - "surface_tools": { - "v2": { - "optimal_calls": 5, - "optimal_tools": {"list_modules", "find_work_items", "archive_work_item"}, - "alternate_tools": { - "get_work_item", - "assign_to_module", - "search_projects", - "list_states", - }, - }, - }, - "needs": {"module"}, - "verify": verify_w5, - }, - { - "id": "W6", - "tags": {"write", "tier1"}, - "prompt": ( - f"In project {{project}}, '{CYCLE_PAST}' is wrapping up. Close it and make sure " - f"its unfinished work items end up on '{CYCLE_CURRENT}'." - ), - "optimal_calls": 4, - "optimal_tools": { - "list_cycles", - "transfer_cycle_work_items", - "complete_cycle", - }, - "alternate_tools": { - "list_cycle_work_items", - "manage_cycle_work_items", - "update_cycle", - "list_work_items", - "list_projects", - }, - "surface_tools": { - "v2": { - # close_cycle with transfer_to is the consolidated path. - "optimal_calls": 2, - "optimal_tools": {"list_cycles", "close_cycle"}, - "alternate_tools": { - "assign_to_cycle", - "find_work_items", - "search_projects", - "get_workspace_context", - }, - }, - }, - # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — - # Plane rejects every edit to an ended cycle. See _seed_cycles. - "needs": {"items", "cycles", "cycles_open_past"}, - "verify": verify_w6, - }, - { - "id": "W7", - "tags": {"write", "tier1"}, - "prompt": ( - f"In project {{project}}, mark the work item '{W7_SOURCE_TITLE}' as blocking " - f"'{W7_TARGET_TITLE}', and add the reference URL {W7_URL} on the blocking item." - ), - "optimal_calls": 3, - "optimal_tools": { - "list_work_items", - "create_work_item_relation", - "create_work_item_link", - }, - "alternate_tools": { - "search_work_items", - "list_work_item_relations", - "list_work_item_relation_definitions", - "list_work_item_links", - "retrieve_work_item", - }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"find_work_items", "link_work_items", "add_work_item_link"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "update_work_item", - }, - }, - }, - "needs": {"items"}, - "verify": verify_w7, - }, - { - "id": "W8", - "tags": {"write", "tier1"}, - "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}' for yesterday."), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "create_work_log"}, - "alternate_tools": { - "search_work_items", - "list_work_logs", - "retrieve_work_item", - "list_projects", - }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "log_work"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "update_work_item", - }, - }, - }, - "needs": {"items"}, - "verify": verify_w8, - }, - { - "id": "W9", - "tags": {"write", "tier1", "extra"}, - "prompt": ( - "In project {project}, set priority to high on these three work items in one " - "batch: 'Checkout times out on 3DS challenge', " - "'Session cookie not rotated after login', " - "'Inventory count goes negative under load'." - ), - # Extra: exercises bulk_update_work_items (not in original DESIGN 20). - "optimal_calls": 4, - "optimal_tools": { - "list_work_items", - "update_work_item", - }, - "alternate_tools": { - "search_work_items", - "list_projects", - "retrieve_work_item", - }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "bulk_update_work_items"}, - "alternate_tools": { - "update_work_item", - "get_work_item", - "search_projects", - }, - }, - }, - "needs": {"items"}, - "verify": verify_w9, - }, - { - "id": "W10", - "tags": {"write", "tier1", "extra"}, - "prompt": ( - "In project {project}, create a project page named 'Eval Runbook' with body " - "text 'Rollback steps for eval harness'." - ), - # Extra: exercises pages family (create_page / get_page). - "optimal_calls": 2, - "optimal_tools": {"list_projects", "create_page"}, - "alternate_tools": { - "list_pages", - "retrieve_page", - "attach_page_to_work_item", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"create_page"}, - "alternate_tools": { - "list_pages", - "get_page", - "search_projects", - "get_workspace_context", - }, - }, - }, - "needs": set(), - "verify": verify_w10, - }, - { - "id": "S1", - "tags": {"setup", "tier1"}, - "prompt": ( - "In project {project}, add a Severity dropdown property (options: Critical, " - "Major, Minor) to the Bug work item type." - ), - "optimal_calls": 3, - "optimal_tools": { - "list_projects", - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "create_work_item_property_option", - "retrieve_work_item_type", - "list_work_item_properties", - "retrieve_work_item_property", - "manage_work_item_type_properties", - "create_work_item_type", - "import_work_item_types_to_project", - "update_project_features", - }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S1 needs work-item-type/property schema tools " - "(resolve_work_item_type, create_work_item_property) which are " - "not on the default v2 surface — use --surface v2-schema" - ), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": { - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "list_work_item_properties", - "add_property_option", - "search_projects", - "get_workspace_context", - "get_features", - "configure_features", - "update_work_item_type", - }, - }, - }, - "needs": {"bug_type"}, - "verify": verify_s1, - }, - { - "id": "S2", - "tags": {"setup", "tier1"}, - "prompt": ( - f"In project {{project}}, add a Fibonacci estimate scale (points 1,2,3,5,8) " - f"and set the work item '{W8_TITLE}' to 5 points." - ), - "optimal_calls": 5, - "optimal_tools": { - "list_projects", - "create_project_estimate", - "create_project_estimate_points", - "link_estimate_to_project", - "update_work_item", - }, - "alternate_tools": { - "get_project_estimate", - "list_project_estimate_points", - "list_work_items", - "search_work_items", - "update_project_estimate", - }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S2 needs configure_estimate (schema tier) to create the Fibonacci " - "scale — default v2 has no estimate schema tools. " - "(v2 update_work_item does accept estimate_point.) Use v2-schema." - ), - }, - "v2-schema": { - # configure_estimate creates scale+points+link in one call; - # update_work_item(estimate_point="5") resolves the value server-side. - "optimal_calls": 2, - "optimal_tools": {"configure_estimate", "update_work_item"}, - "alternate_tools": { - "search_projects", - "get_features", - "find_work_items", - "get_work_item", - "get_workspace_context", - "bulk_update_work_items", - }, - }, - }, - "needs": {"items"}, - "verify": verify_s2, - }, - { - "id": "S3", - "tags": {"setup", "tier1"}, - "prompt": ( - "In project {project}, create a work item type named 'Incident' and add a " - "required text property (e.g. 'Impact summary') on it." - ), - "optimal_calls": 3, - "optimal_tools": { - "list_projects", - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "create_work_item_type", - "list_work_item_types", - "import_work_item_types_to_project", - "list_work_item_properties", - "manage_work_item_type_properties", - "update_project_features", - }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S3 needs resolve_work_item_type + create_work_item_property " - "(schema tier) — use --surface v2-schema" - ), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": { - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "list_work_item_properties", - "update_work_item_type", - "search_projects", - "get_features", - "configure_features", - }, - }, - }, - "needs": set(), - "verify": verify_s3, - }, - { - "id": "S4", - "tags": {"setup", "tier1"}, - "prompt": ( - f"In project {{project}}, triage intake: accept the billing request " - f"'{INTAKE_BILLING_TITLE}' and reject/decline the spam item " - f"'{INTAKE_SPAM_TITLE}'." - ), - "optimal_calls": 3, - "optimal_tools": { - "list_intake_work_items", - "update_intake_work_item", - }, - "alternate_tools": { - "retrieve_intake_work_item", - "list_work_items", - "list_projects", - "create_intake_work_item", - }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"list_intake", "triage_intake"}, - "alternate_tools": { - "find_work_items", - "get_work_item", - "search_projects", - "get_workspace_context", - }, - }, - }, - "needs": {"intake"}, - "verify": verify_s4, - }, - { - "id": "S5", - "tags": {"setup", "tier1"}, - "prompt": ( - "Enable cycles and time tracking (worklogs) for project {project}, " - "and enable the customers feature for the workspace." - ), - # Minimal legacy path (2 calls): - # 1. update_project(cycle_view=True, is_time_tracking_enabled=True) - # 2. update_workspace_features(customers=True) - # (features PATCH can set cycles→cycle_view but cannot set worklogs.) - "optimal_calls": 2, - "optimal_tools": {"update_project", "update_workspace_features"}, - "alternate_tools": { - "update_project_features", - "list_projects", - "retrieve_project", - "get_features", - }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S5 needs configure_features (schema tier) for project cycles/worklogs " - "and workspace customers — use --surface v2-schema" - ), - }, - "v2-schema": { - # 2 calls: configure_features(project, cycles+worklogs) + - # configure_features(customers=True) without project. - "optimal_calls": 2, - "optimal_tools": {"configure_features"}, - "alternate_tools": { - "get_features", - "search_projects", - "get_workspace_context", - "update_work_item", - }, - }, - }, - # Seed leaves project cycles+worklogs and workspace customers off. - "needs": {"leave_cycles_worklogs_off"}, - "verify": verify_s5, - }, - { - "id": "C1", - "tags": {"write", "tier1"}, - "prompt": ( - f"Create customer '{CUSTOMER_NAME}' (if it does not already exist), add a " - f"request named '{CUSTOMER_REQUEST_NAME}', and link that request to the work " - f"item '{R1_TITLE}' in project {{project}}." - ), - "optimal_calls": 4, - "optimal_tools": { - "list_customers", - "create_customer", - "create_customer_request", - "list_work_items", - }, - "alternate_tools": { - "retrieve_customer", - "manage_customer_work_items", - "list_customer_requests", - "list_customer_work_items", - "search_work_items", - "list_projects", - }, - "surface_tools": { - "v2": { - "optimal_calls": 4, - "optimal_tools": { - "list_customers", - "create_customer", - "log_customer_request", - "link_customer_work_items", - }, - "alternate_tools": { - "get_customer", - "find_work_items", - "update_customer", - "search_projects", - }, - }, - }, - # No pre-seeded customer — agent creates; items needed for link target. - "needs": {"items"}, - "verify": verify_c1, - }, - { - "id": "C2", - "tags": {"read", "tier1"}, - "prompt": (f"What shipped in release {RELEASE_NAME}? Summarize the changelog."), - "optimal_calls": 2, - "optimal_tools": {"list_releases", "get_release_changelog"}, - "alternate_tools": { - "retrieve_release", - "list_release_work_items", - "update_release_changelog", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"get_release"}, - "alternate_tools": { - "list_releases", - "assign_to_release", - "get_workspace_context", - }, - }, - }, - "needs": {"release"}, - "verify": verify_c2, - }, - { - "id": "R7", - "tags": {"read", "tier1", "extra"}, - "prompt": ( - f"In project {{project}}, what states can the work item '{R1_TITLE}' " - "legally transition to under workflow rules? List the state names " - "(or say unrestricted if none)." - ), - # Extra: exercises list_available_transitions. - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_states"}, - "alternate_tools": { - "retrieve_work_item", - "search_work_items", - "list_projects", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"list_available_transitions"}, - "alternate_tools": { - "find_work_items", - "get_work_item", - "list_states", - "search_projects", - }, - }, - }, - "needs": {"items"}, - "verify": verify_r7, - }, - # ------------------------------------------------------------------ - # ID-in-hand class (I*): identifiers handed in the prompt — no name resolution advantage - # ------------------------------------------------------------------ - { - "id": "I1", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, - "prompt": ("In project {project}, update work item {work_item_id}: set its priority to high."), - "prompt_bind": _bind_item_uuid(I1_TITLE), - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - "retrieve_work_item_by_identifier", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "find_work_items", "search_projects"}, - }, - }, - "needs": {"items"}, - "verify": verify_i1, - }, - { - "id": "I2", - "author": "post-hoc-debias", - "tags": {"read", "tier1", "id_in_hand", "debias"}, - "prompt": ( - "In project {project}, what is the current state of work item " - "{work_item_identifier}? Answer with the state name only." - ), - "prompt_bind": _bind_item_identifier(I2_TITLE), - "optimal_calls": 1, - "optimal_tools": {"retrieve_work_item_by_identifier"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - "list_states", - }, - "surface_tools": { - "v2": { - # get_work_item requires UUIDs (forwards work_item_id directly). - # PROJ-N on v2 is resolved via find_work_items (list/filter). - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "list_states", - "search_projects", - "get_workspace_context", - }, - }, - }, - "needs": {"items"}, - "verify": verify_i2, - }, - { - "id": "I3", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, - "prompt": ("In project {project}, add work item {work_item_id} to cycle {cycle_id}."), - "prompt_bind": _bind_i3, - "optimal_calls": 1, - "optimal_tools": {"manage_cycle_work_items"}, - "alternate_tools": { - "list_cycles", - "list_cycle_work_items", - "list_work_items", - "retrieve_cycle", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"assign_to_cycle"}, - "alternate_tools": {"list_cycles", "find_work_items", "get_work_item"}, - }, - }, - "needs": {"items", "cycles"}, - "verify": verify_i3, - }, - { - "id": "I4", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, - "prompt": ("In project {project}, attach label {label_id} to work item {work_item_id}."), - "prompt_bind": _bind_i4, - "optimal_calls": 1, - "optimal_tools": {"manage_work_item_label"}, - "alternate_tools": { - "update_work_item", - "list_labels", - "retrieve_work_item", - "list_work_items", - }, - "surface_tools": { - "v2": { - # Default v2 update_work_item accepts labels; no manage_work_item_label. - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "list_labels", "find_work_items"}, - }, - }, - "needs": {"items", "labels"}, - "verify": verify_i4, - }, - { - "id": "I5", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, - "prompt": ("In project {project}, set the priority of work item {work_item_id} to low."), - "prompt_bind": _bind_item_uuid(I3_TITLE), # footer item; not high-traffic elsewhere - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "find_work_items"}, - }, - }, - "needs": {"items"}, - "verify": verify_i5, - }, - # ------------------------------------------------------------------ - # Long-tail class (L*): tools outside the default v2 curated surface - # ------------------------------------------------------------------ - { - "id": "L1", - "author": "post-hoc-debias", - "tags": {"write", "read", "tier1", "long_tail", "debias"}, - "prompt": ( - f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " - f"'{L1_TITLE}', then report the project's worklog summary (who/what has time logged)." - ), - "optimal_calls": 3, - "optimal_tools": {"list_work_items", "create_work_log", "get_project_worklog_summary"}, - "alternate_tools": { - "search_work_items", - "list_work_logs", - "retrieve_work_item", - "list_projects", - }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ( - "L1 needs get_project_worklog_summary (legacy project tool) — not on the default v2 surface" - ), - }, - }, - "needs": {"items"}, - "verify": verify_l1, - }, - { - "id": "L2", - "author": "post-hoc-debias", - "tags": {"read", "tier1", "long_tail", "debias"}, - "prompt": ( - f"In project {{project}}, list the activity history for the work item titled " - f"'{L2_TITLE}'. Summarize how many activities there are and mention any " - "notable comment phrases you see. End your answer with a line of the form " - "'count: N' where N is the number of activities." - ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_activities"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "list_work_item_comments", - "retrieve_work_item_activity", - }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ( - "L2 needs list_work_item_activities — not on the default v2 surface " - "(v2 has comments include= but not the activities feed)" - ), - }, - }, - "needs": {"items", "activity_feed"}, - "verify": verify_l2, - }, - { - "id": "L3", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "long_tail", "debias"}, - "prompt": (f"Create a release tag with version '{L3_TAG_VERSION}' (a version marker for the eval run)."), - "optimal_calls": 1, - "optimal_tools": {"create_release_tag"}, - "alternate_tools": { - "list_release_tags", - "retrieve_release_tag", - "list_releases", - "update_release_tag", - }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": "L3 needs create_release_tag — not on the default v2 surface", - }, - }, - "needs": set(), # workspace-level tag; no project fixture required - "verify": verify_l3, - }, - { - "id": "L4", - "author": "post-hoc-debias", - "tags": {"write", "tier1", "long_tail", "debias"}, - "prompt": ( - f"For customer '{CUSTOMER_NAME}', ensure there is a text customer property " - f"named '{L4_PROP_DISPLAY}' and set its value to '{L4_PROP_VALUE}'." - ), - "optimal_calls": 3, - "optimal_tools": { - "list_customers", - "create_customer_property", - "set_customer_property_values", - }, - "alternate_tools": { - "list_customer_properties", - "get_customer_property_values", - "retrieve_customer", - "update_customer_property", - }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ( - "L4 needs create_customer_property / set_customer_property_values — not on the default v2 surface" - ), - }, - }, - "needs": {"customer"}, - "verify": verify_l4, - }, - { - "id": "L5", - "author": "post-hoc-debias", - "tags": {"read", "tier1", "long_tail", "debias"}, - "prompt": ( - f"In project {{project}}, how many file attachments does the work item titled " - f"'{L5_TITLE}' have? End your answer with a line of the form 'count: N' " - "where N is the number of file attachments." - ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_attachments"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "get_work_item_attachment_download_url", - }, - "surface_tools": { - "v2": { - # Achievable on default v2 via include=attachments on get_work_item. - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "get_work_item"}, - "alternate_tools": { - "search_projects", - "get_workspace_context", - "list_states", - }, - }, - }, - "needs": {"items"}, - "verify": verify_l5, - }, -] - - -def resolve_surface_tool_sets( - task: dict[str, Any], - surface: str, -) -> dict[str, Any]: - """Resolve optimal/alternate tool sets for a surface. - - Returns a dict with: - - skip (str | None): if set, the runner should SKIP the task on this surface - - optimal_tools / alternate_tools: classification sets - - optimal_calls: optional override - - classification: ``exact`` when an overlay or full/legacy sets apply; - ``approximate`` when falling back to flat legacy-named sets on a non-full - surface that has no overlay - """ - surface = (surface or "full").strip().lower() - overlays = task.get("surface_tools") or {} - - if surface in ("full", "legacy", ""): - return { - "skip": None, - "optimal_tools": set(task["optimal_tools"]), - "alternate_tools": set(task["alternate_tools"]), - "optimal_calls": task.get("optimal_calls"), - "classification": "exact", - } - - ov = overlays.get(surface) - # v2-schema is a superset of v2 for *supported* tools, but schema adds none of - # the long-tail APIs (worklog summary, activities, release tags, customer - # property values). Inherit the full v2 overlay — including expected_skip / - # unsupported — when no schema-specific entry exists. - if ov is None and surface == "v2-schema": - ov = overlays.get("v2") - - if ov is None: - return { - "skip": None, - "optimal_tools": set(task["optimal_tools"]), - "alternate_tools": set(task["alternate_tools"]), - "optimal_calls": task.get("optimal_calls"), - "classification": "approximate", - } - - if ov.get("unsupported") or ov.get("expected_skip"): - return { - "skip": ov.get("reason") or f"task {task.get('id')} unsupported on surface {surface}", - "optimal_tools": set(), - "alternate_tools": set(), - "optimal_calls": None, - "classification": "exact", - } - - optimal = set(ov["optimal_tools"]) - alternate = set(ov["alternate_tools"]) - if not optimal.isdisjoint(alternate): - raise ValueError(f"{task.get('id')}/{surface}: optimal/alternate overlap") - return { - "skip": None, - "optimal_tools": optimal, - "alternate_tools": alternate, - "optimal_calls": ov.get("optimal_calls", task.get("optimal_calls")), - "classification": "exact", - } - - -TASKS_BY_ID: dict[str, dict[str, Any]] = {t["id"]: t for t in TASKS} - - -def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: - """Return tasks filtered by id list (None = all).""" - if ids is None: - return list(TASKS) - missing = [i for i in ids if i not in TASKS_BY_ID] - if missing: - raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") - return [TASKS_BY_ID[i] for i in ids] - - -def task_author(task: dict[str, Any]) -> str: - """Return the task author; default ``claude`` when the key is absent.""" - return str(task.get("author") or "claude") - - -def _serialize_surface_tools(surface_tools: dict[str, Any] | None) -> dict[str, Any]: - """Stable JSON-friendly form of a task's surface_tools overlay.""" - if not surface_tools: - return {} - out: dict[str, Any] = {} - for surface in sorted(surface_tools): - ov = surface_tools[surface] or {} - if not isinstance(ov, dict): - out[surface] = ov - continue - entry: dict[str, Any] = {} - for key in sorted(ov): - val = ov[key] - if isinstance(val, set | frozenset): - entry[key] = sorted(val) - elif isinstance(val, list | tuple): - entry[key] = list(val) - else: - entry[key] = val - out[surface] = entry - return out - - -def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: - """Stable short hash of the task battery used for a run. - - SHA-256 (first 12 hex chars) over a canonical serialization of every task - sorted by id: id, prompt, sorted optimal/alternate tools, optimal_calls, - and the surface_tools overlay (sets sorted, keys sorted). - - Ceilings (intentionally *not* covered by the hash): - - Verifier functions and ``needs`` fixtures do not alter the fingerprint — - prompt/tool-set drift is the stability signal, not seed/verify logic. - - The hash covers the *selected* task list: ``--tasks`` subsets produce - different fingerprints than a full-catalog run. - """ - src = list(TASKS if tasks is None else tasks) - payload: list[dict[str, Any]] = [] - for t in sorted(src, key=lambda x: str(x.get("id") or "")): - payload.append( - { - "id": t.get("id"), - "prompt": t.get("prompt"), - "optimal_tools": sorted(t.get("optimal_tools") or []), - "alternate_tools": sorted(t.get("alternate_tools") or []), - "optimal_calls": t.get("optimal_calls"), - "surface_tools": _serialize_surface_tools(t.get("surface_tools")), - } - ) - blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py new file mode 100644 index 00000000..1394b355 --- /dev/null +++ b/evals/tasks/__init__.py @@ -0,0 +1,329 @@ +"""Public task catalog assembled from task-class modules.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from evals.tasks.common import ( + PromptBindError, + TaskSkipped, + as_id, + count_open_urgent, + find_item_by_name, + find_items_by_name, + format_task_prompt, + get_final_text, + ids, + is_not_found, + reports_contract_int, + reports_exact_int, + state_group, + state_name, + whole_answer_int, + word_boundary, +) +from evals.tasks.cross import CROSS_TASKS, verify_c1, verify_c2 +from evals.tasks.debias import ( + DEBIAS_TASKS, + I1_TITLE, + I2_TITLE, + I3_TITLE, + I4_TITLE, + L1_TITLE, + L2_TITLE, + L3_TAG_VERSION, + L4_PROP_DISPLAY, + L4_PROP_VALUE, + L5_TITLE, + verify_i1, + verify_i2, + verify_i3, + verify_i4, + verify_i5, + verify_l1, + verify_l2, + verify_l3, + verify_l4, + verify_l5, +) +from evals.tasks.read import ( + READ_TASKS, + verify_r1, + verify_r2, + verify_r3, + verify_r4, + verify_r5, + verify_r6, + verify_r7, +) +from evals.tasks.schema import SCHEMA_TASKS, verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.write import ( + WRITE_TASKS, + verify_w1, + verify_w2, + verify_w3, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w8, + verify_w9, + verify_w10, +) + +EXPECTED_TASK_IDS = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) + +# Preserve the historical catalog order exactly: R7 was added after C1/C2. +TASKS: list[dict[str, Any]] = [ + *READ_TASKS[:6], + *WRITE_TASKS, + *SCHEMA_TASKS, + *CROSS_TASKS, + READ_TASKS[6], + *DEBIAS_TASKS, +] +if tuple(task["id"] for task in TASKS) != EXPECTED_TASK_IDS: + raise RuntimeError("assembled task order changed; battery/result compatibility would break") + +TASKS_BY_ID: dict[str, dict[str, Any]] = {task["id"]: task for task in TASKS} + + +def resolve_surface_tool_sets( + task: dict[str, Any], + surface: str, +) -> dict[str, Any]: + """Resolve optimal/alternate tool sets for a surface. + + Returns a dict with: + - skip (str | None): if set, the runner should SKIP the task on this surface + - optimal_tools / alternate_tools: classification sets + - optimal_calls: optional override + - classification: ``exact`` when an overlay or full/legacy sets apply; + ``approximate`` when falling back to flat legacy-named sets on a non-full + surface that has no overlay + """ + surface = (surface or "full").strip().lower() + overlays = task.get("surface_tools") or {} + + if surface in ("full", "legacy", ""): + return { + "skip": None, + "optimal_tools": set(task["optimal_tools"]), + "alternate_tools": set(task["alternate_tools"]), + "optimal_calls": task.get("optimal_calls"), + "classification": "exact", + } + + ov = overlays.get(surface) + # v2-schema is a superset of v2 for *supported* tools, but schema adds none of + # the long-tail APIs (worklog summary, activities, release tags, customer + # property values). Inherit the full v2 overlay — including expected_skip / + # unsupported — when no schema-specific entry exists. + if ov is None and surface == "v2-schema": + ov = overlays.get("v2") + + if ov is None: + return { + "skip": None, + "optimal_tools": set(task["optimal_tools"]), + "alternate_tools": set(task["alternate_tools"]), + "optimal_calls": task.get("optimal_calls"), + "classification": "approximate", + } + + if ov.get("unsupported") or ov.get("expected_skip"): + return { + "skip": ov.get("reason") or f"task {task.get('id')} unsupported on surface {surface}", + "optimal_tools": set(), + "alternate_tools": set(), + "optimal_calls": None, + "classification": "exact", + } + + optimal = set(ov["optimal_tools"]) + alternate = set(ov["alternate_tools"]) + if not optimal.isdisjoint(alternate): + raise ValueError(f"{task.get('id')}/{surface}: optimal/alternate overlap") + return { + "skip": None, + "optimal_tools": optimal, + "alternate_tools": alternate, + "optimal_calls": ov.get("optimal_calls", task.get("optimal_calls")), + "classification": "exact", + } + + +def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: + """Return tasks filtered by id list (None = all).""" + if ids is None: + return list(TASKS) + missing = [i for i in ids if i not in TASKS_BY_ID] + if missing: + raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") + return [TASKS_BY_ID[i] for i in ids] + + +def task_author(task: dict[str, Any]) -> str: + """Return the task author; default ``claude`` when the key is absent.""" + return str(task.get("author") or "claude") + + +def _serialize_surface_tools(surface_tools: dict[str, Any] | None) -> dict[str, Any]: + """Stable JSON-friendly form of a task's surface_tools overlay.""" + if not surface_tools: + return {} + out: dict[str, Any] = {} + for surface in sorted(surface_tools): + ov = surface_tools[surface] or {} + if not isinstance(ov, dict): + out[surface] = ov + continue + entry: dict[str, Any] = {} + for key in sorted(ov): + val = ov[key] + if isinstance(val, set | frozenset): + entry[key] = sorted(val) + elif isinstance(val, list | tuple): + entry[key] = list(val) + else: + entry[key] = val + out[surface] = entry + return out + + +def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: + """Stable short hash of the task battery used for a run. + + SHA-256 (first 12 hex chars) over a canonical serialization of every task + sorted by id: id, prompt, sorted optimal/alternate tools, optimal_calls, + and the surface_tools overlay (sets sorted, keys sorted). + + Ceilings (intentionally *not* covered by the hash): + - Verifier functions and ``needs`` fixtures do not alter the fingerprint — + prompt/tool-set drift is the stability signal, not seed/verify logic. + - The hash covers the *selected* task list: ``--tasks`` subsets produce + different fingerprints than a full-catalog run. + """ + src = list(TASKS if tasks is None else tasks) + payload: list[dict[str, Any]] = [] + for t in sorted(src, key=lambda x: str(x.get("id") or "")): + payload.append( + { + "id": t.get("id"), + "prompt": t.get("prompt"), + "optimal_tools": sorted(t.get("optimal_tools") or []), + "alternate_tools": sorted(t.get("alternate_tools") or []), + "optimal_calls": t.get("optimal_calls"), + "surface_tools": _serialize_surface_tools(t.get("surface_tools")), + } + ) + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + +__all__ = [ + "EXPECTED_TASK_IDS", + "PromptBindError", + "TASKS", + "TASKS_BY_ID", + "TaskSkipped", + "as_id", + "battery_fingerprint", + "count_open_urgent", + "find_item_by_name", + "find_items_by_name", + "format_task_prompt", + "get_final_text", + "get_tasks", + "ids", + "is_not_found", + "reports_contract_int", + "reports_exact_int", + "resolve_surface_tool_sets", + "state_group", + "state_name", + "task_author", + "whole_answer_int", + "word_boundary", + "I1_TITLE", + "I2_TITLE", + "I3_TITLE", + "I4_TITLE", + "L1_TITLE", + "L2_TITLE", + "L3_TAG_VERSION", + "L4_PROP_DISPLAY", + "L4_PROP_VALUE", + "L5_TITLE", + "verify_r1", + "verify_r2", + "verify_r3", + "verify_r4", + "verify_r5", + "verify_r6", + "verify_r7", + "verify_w1", + "verify_w2", + "verify_w3", + "verify_w4", + "verify_w5", + "verify_w6", + "verify_w7", + "verify_w8", + "verify_w9", + "verify_w10", + "verify_s1", + "verify_s2", + "verify_s3", + "verify_s4", + "verify_s5", + "verify_c1", + "verify_c2", + "verify_i1", + "verify_i2", + "verify_i3", + "verify_i4", + "verify_i5", + "verify_l1", + "verify_l2", + "verify_l3", + "verify_l4", + "verify_l5", +] diff --git a/evals/tasks/common.py b/evals/tasks/common.py new file mode 100644 index 00000000..2b17b858 --- /dev/null +++ b/evals/tasks/common.py @@ -0,0 +1,272 @@ +"""Shared prompt, matching, and API lookup machinery for eval tasks.""" + +from __future__ import annotations + +import re +import string +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.query_params import WorkItemQueryParams + + +class TaskSkipped(Exception): + """Verifier signals that this task-rep should be recorded as skipped, not failed.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class PromptBindError(RuntimeError): + """Live prompt could not bind required seed IDs (classified as infra_seed).""" + + +def format_task_prompt( + task: dict[str, Any], + ctx: dict[str, Any] | None = None, + *, + strict: bool = False, +) -> str: + """Render a task prompt with seed-bound placeholders. + + Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand + the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an + optional ``prompt_bind(ctx) -> dict`` callable on the task dict. + + When ``strict=True`` (live runs), empty-string values or binder exceptions + raise ``PromptBindError`` so the harness records ``infra_seed`` rather than + sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and + fills missing keys with explicit ```` markers. + """ + tpl = str(task.get("prompt") or "") + fields: dict[str, Any] = { + "project": (ctx or {}).get("project_name") or "EVAL deadbeef", + } + binder = task.get("prompt_bind") + if callable(binder) and ctx is not None: + try: + extra = binder(ctx) or {} + except Exception as exc: + if strict: + raise PromptBindError( + f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" + ) from exc + extra = {} + if isinstance(extra, dict): + for key, val in extra.items(): + if val is None: + if strict: + raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") + continue + text = str(val).strip() + if not text: + if strict: + raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") + continue + fields[key] = text + # Collect required placeholders from the template. + required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] + for name in required: + if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): + continue + if strict: + raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") + fields.setdefault(name, f"<{name}>") + return tpl.format(**fields) + + +def word_boundary(value: str) -> re.Pattern[str]: + """Compile a case-insensitive word-boundary match for an exact seeded value.""" + return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) + + +def reports_exact_int(text: str, n: int) -> bool: + """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" + return bool(word_boundary(str(int(n))).search(text or "")) + + +def whole_answer_int(text: str) -> int | None: + """If the answer (or its last non-empty line) is exactly an integer, return it. + + Letters must not appear — only surrounding whitespace/punctuation is ignored — + so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading + minus** attached to the number is preserved (``-3`` → -3, not 3). + """ + + def _as_int(s: str) -> int | None: + # Collapse whitespace; then the whole string must be optional sign + digits + # with only non-word punctuation wrappers (prefix must not eat the sign). + compact = re.sub(r"\s+", "", s or "") + m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) + if m: + return int(m.group(1)) + return None + + blob = text or "" + v = _as_int(blob) + if v is not None: + return v + lines = [ln for ln in blob.splitlines() if ln.strip()] + if lines: + return _as_int(lines[-1]) + return None + + +def reports_contract_int(text: str, truth: int) -> bool: + """True when final text reports ``truth`` via the explicit ``count: N`` contract. + + 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding + whitespace allowed). Use the **last** match; require signed equality with + ``truth``. + 2. Fallback: whole-answer / last-line bare integer (:func:`whole_answer_int`). + 3. No match at all → False (ignoring an explicit format instruction is a fail). + """ + last: int | None = None + for line in (text or "").splitlines(): + m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) + if m: + last = int(m.group(1)) + if last is not None: + return last == int(truth) + whole = whole_answer_int(text) + if whole is not None: + return whole == int(truth) + return False + + +def as_id(obj: Any) -> str | None: + if obj is None: + return None + if isinstance(obj, str): + return obj + return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) + + +def ids(items: Any) -> set[str]: + out: set[str] = set() + for item in items or []: + i = as_id(item) + if i: + out.add(str(i)) + return out + + +def find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: + """Return all work items with exact name, newest first (by created_at).""" + matches: list[Any] = [] + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in page.results or []: + if (item.name or "").strip() == name: + matches.append(item) + if not page.next_page_results: + break + cursor = page.next_cursor + + def _created_key(item: Any) -> str: + return str(getattr(item, "created_at", None) or "") + + matches.sort(key=_created_key, reverse=True) + return matches + + +def find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: + """Locate a work item by exact name; when duplicates exist, prefer the newest.""" + matches = find_items_by_name(plane, workspace_slug, project_id, name) + return matches[0] if matches else None + + +def state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + """Resolve a state UUID or expanded object to its display name.""" + if state_ref is None: + return None + if hasattr(state_ref, "name") and state_ref.name: + return str(state_ref.name) + if isinstance(state_ref, dict) and state_ref.get("name"): + return str(state_ref["name"]) + state_id = as_id(state_ref) + if not state_id: + return None + try: + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return state.name + except HttpError as exc: + if exc.status_code not in (404, 405): + raise + # Fall back to listing states and matching by id. + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + results = page.results if hasattr(page, "results") else page + for s in results or []: + if str(s.id) == str(state_id): + return s.name + return None + + +def state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + if state_ref is None: + return None + if hasattr(state_ref, "group") and state_ref.group: + return str(state_ref.group) + if isinstance(state_ref, dict) and state_ref.get("group"): + return str(state_ref["group"]) + state_id = as_id(state_ref) + if not state_id: + return None + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + for s in page.results or []: + if str(s.id) == str(state_id): + return getattr(s, "group", None) + return None + + +def is_not_found(exc: BaseException) -> bool: + return isinstance(exc, HttpError) and exc.status_code in (404, 405) + + +def get_final_text(run: dict[str, Any]) -> str: + return run.get("final_text") or "" + + +def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: + """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} + n = 0 + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in resp.results or []: + if (getattr(item, "priority", None) or "").lower() != "urgent": + continue + sid = as_id(item.state) + if sid and str(sid) in closed_ids: + continue + n += 1 + if not resp.next_page_results: + break + cursor = resp.next_cursor + return n + + +__all__ = [ + "TaskSkipped", + "PromptBindError", + "format_task_prompt", + "word_boundary", + "reports_exact_int", + "whole_answer_int", + "reports_contract_int", + "as_id", + "ids", + "find_items_by_name", + "find_item_by_name", + "state_name", + "state_group", + "is_not_found", + "get_final_text", + "count_open_urgent", +] diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py new file mode 100644 index 00000000..6bd7ee0b --- /dev/null +++ b/evals/tasks/cross.py @@ -0,0 +1,188 @@ +"""Cross-entity task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from evals.seed import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + R1_TITLE, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, +) +from evals.tasks.common import as_id, find_item_by_name, get_final_text, ids, word_boundary + + +async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C1: customer 'Acme Corp' has request 'SSO support' linked to the R1 work item. + + Anchors: exact customer name, exact request name, and the R1_TITLE work item id + resolved from the eval project at verify time (must be among linked ids). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve the required link target first (seeded Payment webhook item). + r1 = find_item_by_name(plane, workspace_slug, project_id, R1_TITLE) + if r1 is None: + return False, f"R1 item {R1_TITLE!r} not found in project" + + customers = plane.customers.list(workspace_slug=workspace_slug) + rows = customers.results if hasattr(customers, "results") else customers + # Exact name only — do not match arbitrary acme* customers. + acme = next((c for c in (rows or []) if (c.name or "").strip() == CUSTOMER_NAME), None) + if acme is None: + return False, f"customer {CUSTOMER_NAME!r} not found" + + # Track for teardown if agent-created + if not ctx.get("customer"): + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": acme.id}) + + reqs = plane.customers.requests.list(workspace_slug=workspace_slug, customer_id=acme.id) + rrows = reqs.results if hasattr(reqs, "results") else reqs + sso = next( + (r for r in (rrows or []) if (r.name or "").strip() == CUSTOMER_REQUEST_NAME), + None, + ) + if sso is None: + ok = False + notes.append(f"request {CUSTOMER_REQUEST_NAME!r} missing") + else: + notes.append("SSO request present") + + # Require the R1 work item among customer-linked work items. + try: + wi = plane.customers.work_items.list(workspace_slug=workspace_slug, customer_id=acme.id) + wi_rows = list(wi.results if hasattr(wi, "results") else wi or []) + linked_ids = ids(wi_rows) + # Plain string ids also count. + for row in wi_rows: + if isinstance(row, str): + linked_ids.add(row) + elif isinstance(row, dict) and row.get("id"): + linked_ids.add(str(row["id"])) + else: + # Customer work item wrappers may expose work_item / issue field. + for attr in ("work_item", "work_item_id", "issue", "issue_id"): + ref = getattr(row, attr, None) if not isinstance(row, dict) else row.get(attr) + rid = as_id(ref) + if rid: + linked_ids.add(str(rid)) + if str(r1.id) not in linked_ids: + ok = False + notes.append(f"R1 item {r1.id} not linked; linked={sorted(linked_ids)}") + else: + notes.append(f"R1 item {r1.id} linked") + except Exception as exc: + ok = False + notes.append(f"list customer work items failed: {exc}") + + return ok, "; ".join(notes) + + +C1_TASK: dict[str, Any] = { + "id": "C1", + "tags": {"write", "tier1"}, + "prompt": ( + f"Create customer '{CUSTOMER_NAME}' (if it does not already exist), add a " + f"request named '{CUSTOMER_REQUEST_NAME}', and link that request to the work " + f"item '{R1_TITLE}' in project {{project}}." + ), + "optimal_calls": 4, + "optimal_tools": { + "list_customers", + "create_customer", + "create_customer_request", + "list_work_items", + }, + "alternate_tools": { + "retrieve_customer", + "manage_customer_work_items", + "list_customer_requests", + "list_customer_work_items", + "search_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 4, + "optimal_tools": { + "list_customers", + "create_customer", + "log_customer_request", + "link_customer_work_items", + }, + "alternate_tools": { + "get_customer", + "find_work_items", + "update_customer", + "search_projects", + }, + }, + }, + # No pre-seeded customer — agent creates; items needed for link target. + "needs": {"items"}, + "verify": verify_c1, +} + + +async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C2: final text mentions release 1.2.0 and at least one seeded changelog phrase.""" + final_text = get_final_text(run) + notes: list[str] = [] + ok = True + if not word_boundary(RELEASE_NAME).search(final_text): + ok = False + notes.append(f"missing release name {RELEASE_NAME!r}") + else: + notes.append(f"names {RELEASE_NAME}") + changelog = ctx.get("release_changelog_text") or RELEASE_CHANGELOG_TEXT + # Match distinctive fragments from the seeded changelog. + fragments = ["OAuth login hardening", "webhook retry backoff"] + hit = [f for f in fragments if word_boundary(f).search(final_text)] + if not hit: + # Also accept substring of full changelog without word-boundary if short. + if changelog[:40].casefold() not in final_text.casefold(): + ok = False + notes.append("missing changelog content") + else: + notes.append("changelog substring present") + else: + notes.append(f"changelog phrases {hit}") + return ok, "; ".join(notes) + + +C2_TASK: dict[str, Any] = { + "id": "C2", + "tags": {"read", "tier1"}, + "prompt": (f"What shipped in release {RELEASE_NAME}? Summarize the changelog."), + "optimal_calls": 2, + "optimal_tools": {"list_releases", "get_release_changelog"}, + "alternate_tools": { + "retrieve_release", + "list_release_work_items", + "update_release_changelog", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"get_release"}, + "alternate_tools": { + "list_releases", + "assign_to_release", + "get_workspace_context", + }, + }, + }, + "needs": {"release"}, + "verify": verify_c2, +} + + +CROSS_TASKS: list[dict[str, Any]] = [C1_TASK, C2_TASK] + + +__all__ = ["CROSS_TASKS", "verify_c1", "verify_c2"] diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py new file mode 100644 index 00000000..2211e17e --- /dev/null +++ b/evals/tasks/debias.py @@ -0,0 +1,717 @@ +"""ID-in-hand and long-tail de-biasing tasks with their verifiers.""" + +from __future__ import annotations + +import re +from typing import Any + +from plane.models.query_params import RetrieveQueryParams + +from evals.seed import ( + CUSTOMER_NAME, + CYCLE_CURRENT, + DEBIAS_CUSTOMER_PROP_DISPLAY, + DEBIAS_RELEASE_TAG_VERSION, + R1_TITLE, + R5_TITLE, + W2_TITLE, + W3_TITLE, + W8_TITLE, +) +from evals.tasks.common import get_final_text, ids, reports_contract_int, state_name, word_boundary + +I1_TITLE = R1_TITLE + + +I2_TITLE = W2_TITLE + + +I3_TITLE = "Footer year still says 2024" + + +I4_TITLE = W3_TITLE + + +L1_TITLE = W8_TITLE + + +L2_TITLE = R5_TITLE + + +L5_TITLE = R1_TITLE + + +L3_TAG_VERSION = DEBIAS_RELEASE_TAG_VERSION + + +L4_PROP_DISPLAY = DEBIAS_CUSTOMER_PROP_DISPLAY + + +L4_PROP_VALUE = "Enterprise" + + +def _bind_item_uuid(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(title) or "") + return {"work_item_id": wid} + + return _bind + + +def _bind_item_identifier(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + ident = str((ctx.get("item_identifiers") or {}).get(title) or "") + return {"work_item_identifier": ident} + + return _bind + + +def _bind_i3(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + return {"work_item_id": wid, "cycle_id": cycle_id} + + +def _bind_i4(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I4_TITLE) or "") + label_id = str((ctx.get("labels") or {}).get("perf") or "") + return {"work_item_id": wid, "label_id": label_id} + + +async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I1: seeded R1 item priority is high (updated by UUID, not name).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I1_TITLE) + if not wid: + return False, f"seed item {I1_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "high": + return True, f"work_item {wid} priority=high" + return False, f"work_item {wid} priority={pr!r} (want high)" + + +I1_TASK: dict[str, Any] = { + "id": "I1", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, update work item {work_item_id}: set its priority to high."), + "prompt_bind": _bind_item_uuid(I1_TITLE), + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + "retrieve_work_item_by_identifier", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "find_work_items", "search_projects"}, + }, + }, + "needs": {"items"}, + "verify": verify_i1, +} + + +async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I2: final text names the state of the identifier-target item.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I2_TITLE) + if not wid: + return False, f"seed item {I2_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + name = state_name(plane, workspace_slug, project_id, detail.state) + if not name: + return False, "target state name unresolved" + final_text = get_final_text(run) + if word_boundary(name).search(final_text): + return True, f"final text names state {name!r}" + return False, f"final text missing state {name!r}" + + +I2_TASK: dict[str, Any] = { + "id": "I2", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "id_in_hand", "debias"}, + "prompt": ( + "In project {project}, what is the current state of work item " + "{work_item_identifier}? Answer with the state name only." + ), + "prompt_bind": _bind_item_identifier(I2_TITLE), + "optimal_calls": 1, + "optimal_tools": {"retrieve_work_item_by_identifier"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + "list_states", + }, + "surface_tools": { + "v2": { + # get_work_item requires UUIDs (forwards work_item_id directly). + # PROJ-N on v2 is resolved via find_work_items (list/filter). + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "list_states", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"items"}, + "verify": verify_i2, +} + + +async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I3: work item UUID is on the target cycle UUID.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + if not wid or not cycle_id: + return False, "seed work_item_id/cycle_id missing" + page = plane.cycles.list_work_items(workspace_slug=workspace_slug, project_id=project_id, cycle_id=cycle_id) + rows = page.results if hasattr(page, "results") else page + ids = {str(getattr(r, "id", None) or r) for r in (rows or [])} + # list may return issue wrappers with issue/id fields + for r in rows or []: + for attr in ("id", "issue", "work_item_id"): + v = getattr(r, attr, None) + if v is not None: + ids.add(str(v if not hasattr(v, "id") else v.id)) + if wid in ids: + return True, f"item {wid} on cycle {cycle_id}" + return False, f"item {wid} not on cycle {cycle_id}; have {sorted(ids)[:12]}" + + +I3_TASK: dict[str, Any] = { + "id": "I3", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, add work item {work_item_id} to cycle {cycle_id}."), + "prompt_bind": _bind_i3, + "optimal_calls": 1, + "optimal_tools": {"manage_cycle_work_items"}, + "alternate_tools": { + "list_cycles", + "list_cycle_work_items", + "list_work_items", + "retrieve_cycle", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"assign_to_cycle"}, + "alternate_tools": {"list_cycles", "find_work_items", "get_work_item"}, + }, + }, + "needs": {"items", "cycles"}, + "verify": verify_i3, +} + + +async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I4: work item has the seeded perf label id attached.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I4_TITLE) + label_id = (ctx.get("labels") or {}).get("perf") + if not wid or not label_id: + return False, "seed work_item_id/label_id missing" + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=wid, + params=RetrieveQueryParams(expand="labels"), + ) + label_ids = ids(detail.labels) + if str(label_id) in label_ids: + return True, f"label {label_id} on {wid}" + return False, f"labels={sorted(label_ids)} missing perf={label_id}" + + +I4_TASK: dict[str, Any] = { + "id": "I4", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, attach label {label_id} to work item {work_item_id}."), + "prompt_bind": _bind_i4, + "optimal_calls": 1, + "optimal_tools": {"manage_work_item_label"}, + "alternate_tools": { + "update_work_item", + "list_labels", + "retrieve_work_item", + "list_work_items", + }, + "surface_tools": { + "v2": { + # Default v2 update_work_item accepts labels; no manage_work_item_label. + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "list_labels", "find_work_items"}, + }, + }, + "needs": {"items", "labels"}, + "verify": verify_i4, +} + + +async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I5: target item priority is low (updated by UUID).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I3_TITLE) + if not wid: + return False, f"seed item {I3_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "low": + return True, f"work_item {wid} priority=low" + return False, f"work_item {wid} priority={pr!r} (want low)" + + +I5_TASK: dict[str, Any] = { + "id": "I5", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "id_in_hand", "debias"}, + "prompt": ("In project {project}, set the priority of work item {work_item_id} to low."), + "prompt_bind": _bind_item_uuid(I3_TITLE), # footer item; not high-traffic elsewhere + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": { + "retrieve_work_item", + "list_work_items", + "search_work_items", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"update_work_item"}, + "alternate_tools": {"get_work_item", "find_work_items"}, + }, + }, + "needs": {"items"}, + "verify": verify_i5, +} + + +def _l1_duration_reported(final_text: str) -> bool: + """Numeric duration only: whole-word 90 or 1.5 (not English 'ninety').""" + return bool(word_boundary("90").search(final_text)) or bool(re.search(r"\b1\.5\b", final_text)) + + +def _l1_person_names_from_summary(sum_rows: Any) -> list[str]: + """Best-effort actor/assignee display strings from project worklog summary rows.""" + names: list[str] = [] + for row in sum_rows or []: + dump = row.model_dump() if hasattr(row, "model_dump") else {} + if not isinstance(dump, dict): + dump = {} + candidates: list[Any] = [] + for attr in ( + "actor", + "user", + "display_name", + "owned_by", + "created_by", + "assignee", + "email", + "first_name", + "last_name", + ): + v = getattr(row, attr, None) + if v is None and dump: + v = dump.get(attr) + if v is None: + continue + if hasattr(v, "display_name") or hasattr(v, "email"): + candidates.append( + getattr(v, "display_name", None) or getattr(v, "email", None) or getattr(v, "id", None) + ) + elif isinstance(v, dict): + candidates.append(v.get("display_name") or v.get("email") or v.get("id")) + else: + candidates.append(v) + for c in candidates: + s = str(c or "").strip() + if s and s not in names: + names.append(s) + return names + + +def _l1_summary_substance(final_text: str, *, title: str, sum_rows: Any) -> bool: + """Summary half of L1: item title, person from summary, or words summary/total. + + Deliberately does *not* accept bare 'logged' / 'worklog' — the prompt asks to + report the project worklog summary (who/what has time logged). + """ + low = final_text.casefold() + if "summary" in low or "total" in low: + return True + if title and word_boundary(title).search(final_text): + return True + for person in _l1_person_names_from_summary(sum_rows): + if len(person) >= 2 and word_boundary(person).search(final_text): + return True + return False + + +async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L1: 90-minute work log on the correct item AND final text reports duration + summary. + + Duration: numeric whole-word ``90`` or ``1.5`` only (not English 'ninety'). + Summary substance: item title, a person/assignee from the project summary, or + the words ``summary`` / ``total``. Bare "90 minutes of work" fails by design. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L1_TITLE) + if not wid: + return False, f"seed item {L1_TITLE!r} missing" + # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). + logs = plane.work_items.work_logs.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 90 not in durations: + return False, f"no 90-minute work log on target item {wid}; durations={durations}" + + sum_rows: list[Any] = [] + try: + summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) + raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) + sum_rows = list(raw or []) + except Exception: + # Summary fetch is optional for person names; duration + title/summary/total still work. + sum_rows = [] + + final_text = get_final_text(run) + if not _l1_duration_reported(final_text): + return False, "final text missing logged duration (numeric 90 or 1.5)" + if not _l1_summary_substance(final_text, title=L1_TITLE, sum_rows=sum_rows): + return False, ( + "final text lacks worklog summary substance " + "(need item title, person from summary, or words 'summary'/'total')" + ) + return True, f"90m log on {wid} + final text reports duration and summary substance" + + +L1_TASK: dict[str, Any] = { + "id": "L1", + "author": "post-hoc-debias", + "tags": {"write", "read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " + f"'{L1_TITLE}', then report the project's worklog summary (who/what has time logged)." + ), + "optimal_calls": 3, + "optimal_tools": {"list_work_items", "create_work_log", "get_project_worklog_summary"}, + "alternate_tools": { + "search_work_items", + "list_work_logs", + "retrieve_work_item", + "list_projects", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ("L1 needs get_project_worklog_summary (legacy project tool) — not on the default v2 surface"), + }, + }, + "needs": {"items"}, + "verify": verify_l1, +} + + +async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L2: target has activities; final text reports the count via ``count: N`` contract.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L2_TITLE) + if not wid: + return False, f"seed item {L2_TITLE!r} missing" + try: + page = plane.work_items.activities.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except Exception as exc: + return False, f"activities.list failed: {exc}" + rows = page.results if hasattr(page, "results") else page + n = len(list(rows or [])) + if n < 1: + return False, "no activities on target (seed comments should create some)" + final_text = get_final_text(run) + if not reports_contract_int(final_text, n): + return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + return True, f"final text reports activity count {n} via contract" + + +L2_TASK: dict[str, Any] = { + "id": "L2", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, list the activity history for the work item titled " + f"'{L2_TITLE}'. Summarize how many activities there are and mention any " + "notable comment phrases you see. End your answer with a line of the form " + "'count: N' where N is the number of activities." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_activities"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "list_work_item_comments", + "retrieve_work_item_activity", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ( + "L2 needs list_work_item_activities — not on the default v2 surface " + "(v2 has comments include= but not the activities feed)" + ), + }, + }, + "needs": {"items", "activity_feed"}, + "verify": verify_l2, +} + + +async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L3: workspace has a release tag with version eval-rc1.""" + workspace_slug = ctx["workspace_slug"] + try: + page = plane.releases.tags.list(workspace_slug=workspace_slug) + except Exception as exc: + return False, f"list release tags failed: {exc}" + rows = page.results if hasattr(page, "results") else page + versions = {(getattr(t, "version", None) or "").strip() for t in (rows or [])} + if L3_TAG_VERSION in versions: + # Track for teardown if id available. + for t in rows or []: + if (getattr(t, "version", None) or "").strip() == L3_TAG_VERSION: + tid = getattr(t, "id", None) + if tid: + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "release_tag" and str(o.get("id")) == str(tid) for o in objs): + objs.append({"kind": "release_tag", "id": tid}) + break + return True, f"release tag {L3_TAG_VERSION!r} present" + return False, f"tag {L3_TAG_VERSION!r} missing; have {sorted(versions)}" + + +L3_TASK: dict[str, Any] = { + "id": "L3", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "long_tail", "debias"}, + "prompt": (f"Create a release tag with version '{L3_TAG_VERSION}' (a version marker for the eval run)."), + "optimal_calls": 1, + "optimal_tools": {"create_release_tag"}, + "alternate_tools": { + "list_release_tags", + "retrieve_release_tag", + "list_releases", + "update_release_tag", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": "L3 needs create_release_tag — not on the default v2 surface", + }, + }, + "needs": set(), # workspace-level tag; no project fixture required + "verify": verify_l3, +} + + +def _property_type_is_text(prop: Any) -> bool: + raw = getattr(prop, "property_type", None) + if raw is None: + raw = getattr(prop, "type", None) + if raw is None: + return False + if hasattr(raw, "value"): + raw = raw.value + if hasattr(raw, "name"): + # Enum member: PropertyType.TEXT + name = str(raw.name) + if name.upper() == "TEXT": + return True + s = str(raw).upper() + return s == "TEXT" or s.endswith(".TEXT") or s == "PROPERTYTYPE.TEXT" + + +async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L4: right customer has TEXT property 'Eval Industry' = Enterprise (exact name).""" + workspace_slug = ctx["workspace_slug"] + cust = ctx.get("customer") or {} + customer_id = cust.get("id") if isinstance(cust, dict) else cust + if not customer_id: + return False, "customer missing from seed" + try: + props = plane.customers.properties.list(workspace_slug=workspace_slug) + except Exception as exc: + return False, f"list customer properties failed: {exc}" + prop_rows = props.results if hasattr(props, "results") else props + target_prop: Any | None = None + for p in prop_rows or []: + # Exact display_name match only (case-insensitive full match) — not substring "Industry". + display = (getattr(p, "display_name", None) or "").strip() + if display.casefold() != L4_PROP_DISPLAY.casefold(): + continue + if not _property_type_is_text(p): + return False, ( + f"property {display!r} exists but property_type is not TEXT (got {getattr(p, 'property_type', None)!r})" + ) + target_prop = p + break + if target_prop is None: + return False, f"no TEXT customer property named exactly {L4_PROP_DISPLAY!r}" + pid = str(target_prop.id) + # Track for teardown. + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "customer_property" and str(o.get("id")) == pid for o in objs): + objs.append({"kind": "customer_property", "id": pid}) + try: + values = plane.customers.property_values.list(workspace_slug=workspace_slug, customer_id=customer_id) + except Exception as exc: + return False, f"get property values failed: {exc}" + if not isinstance(values, dict): + return False, f"unexpected property_values shape: {type(values)}" + vals = values.get(pid) or values.get(str(pid)) or [] + flat = [str(v) for v in (vals if isinstance(vals, list) else [vals])] + if any(L4_PROP_VALUE.casefold() == v.casefold() for v in flat): + return True, f"customer {customer_id} property {pid} ({L4_PROP_DISPLAY})={L4_PROP_VALUE!r}" + return False, f"customer {customer_id} property {pid} values {flat} lack {L4_PROP_VALUE!r}" + + +L4_TASK: dict[str, Any] = { + "id": "L4", + "author": "post-hoc-debias", + "tags": {"write", "tier1", "long_tail", "debias"}, + "prompt": ( + f"For customer '{CUSTOMER_NAME}', ensure there is a text customer property " + f"named '{L4_PROP_DISPLAY}' and set its value to '{L4_PROP_VALUE}'." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_customers", + "create_customer_property", + "set_customer_property_values", + }, + "alternate_tools": { + "list_customer_properties", + "get_customer_property_values", + "retrieve_customer", + "update_customer_property", + }, + "surface_tools": { + "v2": { + "expected_skip": True, + "reason": ( + "L4 needs create_customer_property / set_customer_property_values — not on the default v2 surface" + ), + }, + }, + "needs": {"customer"}, + "verify": verify_l4, +} + + +async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L5: final text reports the attachment count via ``count: N`` contract.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L5_TITLE) + if not wid: + return False, f"seed item {L5_TITLE!r} missing" + try: + page = plane.work_items.attachments.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except Exception as exc: + return False, f"attachments.list failed: {exc}" + rows = page.results if hasattr(page, "results") else page + n = len(list(rows or [])) + final_text = get_final_text(run) + if not reports_contract_int(final_text, n): + return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + return True, f"final text reports attachment count {n} via contract" + + +L5_TASK: dict[str, Any] = { + "id": "L5", + "author": "post-hoc-debias", + "tags": {"read", "tier1", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, how many file attachments does the work item titled " + f"'{L5_TITLE}' have? End your answer with a line of the form 'count: N' " + "where N is the number of file attachments." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_attachments"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "get_work_item_attachment_download_url", + }, + "surface_tools": { + "v2": { + # Achievable on default v2 via include=attachments on get_work_item. + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "get_work_item"}, + "alternate_tools": { + "search_projects", + "get_workspace_context", + "list_states", + }, + }, + }, + "needs": {"items"}, + "verify": verify_l5, +} + + +DEBIAS_TASKS: list[dict[str, Any]] = [ + I1_TASK, + I2_TASK, + I3_TASK, + I4_TASK, + I5_TASK, + L1_TASK, + L2_TASK, + L3_TASK, + L4_TASK, + L5_TASK, +] + + +__all__ = [ + "DEBIAS_TASKS", + "I1_TITLE", + "I2_TITLE", + "I3_TITLE", + "I4_TITLE", + "L1_TITLE", + "L2_TITLE", + "L3_TAG_VERSION", + "L4_PROP_DISPLAY", + "L4_PROP_VALUE", + "L5_TITLE", + "verify_i1", + "verify_i2", + "verify_i3", + "verify_i4", + "verify_i5", + "verify_l1", + "verify_l2", + "verify_l3", + "verify_l4", + "verify_l5", +] diff --git a/evals/tasks/read.py b/evals/tasks/read.py new file mode 100644 index 00000000..656d1a2c --- /dev/null +++ b/evals/tasks/read.py @@ -0,0 +1,395 @@ +"""Read-task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, R5_TITLE +from evals.tasks.common import ( + count_open_urgent, + find_item_by_name, + get_final_text, + state_name, + word_boundary, +) + + +async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R1: final text must name the target item's state and no other seeded state. + + Matching rule: word-boundary, case-insensitive regex on the exact state name + resolved from the API at verify time (never hardcoded). Additionally fail if + any *other* project state name also matches (blocks guessing/list_states echo). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + title = R1_TITLE + item = find_item_by_name(plane, workspace_slug, project_id, title) + if item is None: + return False, f"seeded item {title!r} not found" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + expected = state_name(plane, workspace_slug, project_id, detail.state) + if not expected: + # Prefer the seeded name when API is sparse. + expected = ctx.get("r1_state_name") + if not expected: + return False, "could not resolve expected state name from API" + + final_text = get_final_text(run) + if not word_boundary(expected).search(final_text): + return False, f"final text missing state name {expected!r}" + + other_states = [n for n in (ctx.get("state_names") or []) if n and n.casefold() != expected.casefold()] + collisions = [n for n in other_states if word_boundary(n).search(final_text)] + if collisions: + return ( + False, + f"final text names other state(s) {collisions!r} besides expected {expected!r}", + ) + return True, f"final text names only state {expected!r}" + + +R1_TASK: dict[str, Any] = { + "id": "R1", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, what is the current state of the work item titled " + f"'{R1_TITLE}'? Answer with the state name." + ), + "optimal_calls": 1, + "optimal_tools": {"list_work_items"}, + "alternate_tools": { + "search_work_items", + "list_archived_work_items", + "count_work_items", + "retrieve_work_item", + "retrieve_work_item_by_identifier", + "list_projects", + "list_states", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "list_states", + "get_workspace_context", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r1, +} + + +async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R2: final text must contain the exact urgent-open count (word-boundary).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + expected = count_open_urgent(plane, workspace_slug, project_id) + final_text = get_final_text(run) + # Word-boundary on the decimal form of the count (blocks "4" matching "24"). + if not word_boundary(str(expected)).search(final_text): + return False, f"final text missing urgent-open count {expected}" + return True, f"final text names count {expected}" + + +R2_TASK: dict[str, Any] = { + "id": "R2", + "tags": {"read", "tier1"}, + "prompt": ("In project {project}, how many urgent open work items are there? Answer with the integer count only."), + "optimal_calls": 1, + "optimal_tools": {"count_work_items"}, + "alternate_tools": { + "list_work_items", + "search_work_items", + "list_projects", + "list_states", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + # No count tool on v2 — find_work_items with priority/state filters is optimal. + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "list_states", + "get_workspace_context", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r2, +} + + +async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R3: final text must include each seeded assigned-to-me / due-this-week title.""" + titles = list(ctx.get("r3_due_titles") or []) + if not titles: + return False, "no R3 due titles in seed ctx" + final_text = get_final_text(run) + missing = [t for t in titles if not word_boundary(t).search(final_text)] + if missing: + return False, f"final text missing title(s) {missing!r}" + return True, f"final text names {len(titles)} due-this-week assigned items" + + +R3_TASK: dict[str, Any] = { + "id": "R3", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, list work items assigned to me that are due this week. Answer with their titles." + ), + "optimal_calls": 2, + "optimal_tools": {"get_me", "list_work_items"}, + "alternate_tools": { + "search_work_items", + "count_work_items", + "list_projects", + "get_workspace_members", + "get_pql_reference", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "get_workspace_context", + "get_work_item", + "search_projects", + "get_pql_reference", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r3, +} + + +async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R4: final text must mention the active cycle name and the overdue item title.""" + final_text = get_final_text(run) + notes: list[str] = [] + ok = True + if not word_boundary(CYCLE_CURRENT).search(final_text): + ok = False + notes.append(f"missing active cycle {CYCLE_CURRENT!r}") + else: + notes.append(f"names {CYCLE_CURRENT}") + overdue = ctx.get("r4_overdue_title") + if overdue: + if not word_boundary(overdue).search(final_text): + # Soft: also accept "overdue" keyword + any active item title. + if "overdue" not in final_text.casefold(): + ok = False + notes.append(f"missing overdue title {overdue!r}") + else: + notes.append("mentions overdue (title not exact)") + else: + notes.append(f"names overdue {overdue!r}") + return ok, "; ".join(notes) + + +R4_TASK: dict[str, Any] = { + "id": "R4", + "tags": {"read", "tier1"}, + "prompt": ( + "In project {project}, what is in the active cycle, and is anything overdue? " + f"Name the cycle (expect '{CYCLE_CURRENT}') and any overdue item titles." + ), + "optimal_calls": 2, + "optimal_tools": {"list_cycles", "list_work_items"}, + "alternate_tools": { + "list_cycle_work_items", + "retrieve_cycle", + "search_work_items", + "list_projects", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"find_work_items"}, + "alternate_tools": { + "list_cycles", + "get_work_item", + "get_pql_reference", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"items", "cycles"}, + "verify": verify_r4, +} + + +async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R5: final text must include seeded comment phrases (word-boundary).""" + phrases = list(ctx.get("r5_comment_phrases") or R5_COMMENT_PHRASES) + final_text = get_final_text(run) + missing = [p for p in phrases if not word_boundary(p).search(final_text)] + if missing: + return False, f"final text missing comment phrase(s) {missing!r}" + return True, f"final text names {len(phrases)} discussion phrases" + + +R5_TASK: dict[str, Any] = { + "id": "R5", + "tags": {"read", "tier1"}, + "prompt": ( + f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " + "Include the key phrases from its comments." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_work_item_comments"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "retrieve_work_item_by_identifier", + "list_work_item_activities", + "list_projects", + }, + "surface_tools": { + "v2": { + # include= depth: single get_work_item with include=comments after resolve, + # or find + get with include. + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "get_work_item"}, + "alternate_tools": { + "search_projects", + "get_workspace_context", + "create_comment", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r5, +} + + +async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R6: final text must name the project that has more open bugs (resolved at verify).""" + expected = ctx.get("r6_more_bugs_project") or ctx.get("second_project_name") + if not expected: + return False, "second project name missing from seed ctx" + final_text = get_final_text(run) + # Match the full project name or the distinctive " B" suffix run8 form. + if word_boundary(expected).search(final_text): + return True, f"final text names project with more bugs {expected!r}" + # Allow matching just the identifier-ish trailing token (e.g. run8 + B). + run8 = ctx.get("run8") or "" + alt = f"EVAL {run8} B" + if word_boundary(alt).search(final_text) or (run8 and run8 in final_text and " B" in final_text): + return True, f"final text names second project ({alt})" + return False, f"final text missing project with more bugs {expected!r}" + + +R6_TASK: dict[str, Any] = { + "id": "R6", + "tags": {"read", "tier1"}, + "prompt": ( + "Across the eval projects created for this run (main project {project} and its " + "sibling 'B' project), which project has more open Bug-typed work items? " + "Answer with the project name." + ), + "optimal_calls": 3, + "optimal_tools": {"list_projects", "list_work_items", "resolve_work_item_type"}, + "alternate_tools": { + "count_work_items", + "search_work_items", + "list_work_item_types", + "retrieve_project", + "get_pql_reference", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"search_projects", "find_work_items", "get_workspace_context"}, + "alternate_tools": { + "get_work_item", + "get_pql_reference", + "list_states", + }, + }, + # Type id resolution cleaner on v2-schema + "v2-schema": { + "optimal_calls": 3, + "optimal_tools": {"search_projects", "find_work_items", "resolve_work_item_type"}, + "alternate_tools": { + "list_work_item_types", + "get_workspace_context", + "get_work_item", + "get_pql_reference", + }, + }, + }, + "needs": {"items", "bug_type", "second_project"}, + "verify": verify_r6, +} + + +async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R7 (extra): final text names at least one legal next state for the R1 item. + + Resolves available completed/started/unstarted states at verify time and + requires a word-boundary hit on one of them (or explicit 'unrestricted'). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + names = [(s.name or "").strip() for s in (page.results or []) if (s.name or "").strip()] + final_text = get_final_text(run) + if "unrestricted" in final_text.casefold() or "any state" in final_text.casefold(): + return True, "agent reported unrestricted transitions" + hits = [n for n in names if word_boundary(n).search(final_text)] + if not hits: + return False, f"final text names none of project states {names}" + return True, f"final text names state(s) {hits}" + + +R7_TASK: dict[str, Any] = { + "id": "R7", + "tags": {"read", "tier1", "extra"}, + "prompt": ( + f"In project {{project}}, what states can the work item '{R1_TITLE}' " + "legally transition to under workflow rules? List the state names " + "(or say unrestricted if none)." + ), + # Extra: exercises list_available_transitions. + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "list_states"}, + "alternate_tools": { + "retrieve_work_item", + "search_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"list_available_transitions"}, + "alternate_tools": { + "find_work_items", + "get_work_item", + "list_states", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_r7, +} + + +READ_TASKS: list[dict[str, Any]] = [R1_TASK, R2_TASK, R3_TASK, R4_TASK, R5_TASK, R6_TASK, R7_TASK] + + +__all__ = ["READ_TASKS", "verify_r1", "verify_r2", "verify_r3", "verify_r4", "verify_r5", "verify_r6", "verify_r7"] diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py new file mode 100644 index 00000000..e740d02a --- /dev/null +++ b/evals/tasks/schema.py @@ -0,0 +1,540 @@ +"""Schema-task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.enums import PropertyType + +from evals.seed import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE +from evals.tasks.common import TaskSkipped, as_id, find_item_by_name, is_not_found + + +async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S1: Bug type has an OPTION property 'Severity' with Critical/Major/Minor. + + Only type-scoped property listing is accepted (no project/workspace fallbacks that + would pass an unattached Severity). Unexpected API errors propagate as harness errors. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + bug_type_id = ( + (ctx.get("bug_type") or {}).get("id") if isinstance(ctx.get("bug_type"), dict) else ctx.get("bug_type") + ) + if not bug_type_id: + raise TaskSkipped("bug_type not seeded") + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + except HttpError as exc: + if is_not_found(exc): + return False, "Severity property not found on Bug type (type-scoped list empty/404)" + raise + + severity = None + for p in props: + display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() + if display.lower() == "severity": + # Prefer an explicit type link when the API exposes it. + issue_type = getattr(p, "issue_type", None) + if issue_type is not None and str(issue_type) not in ("", str(bug_type_id)): + continue + severity = p + break + if severity is None: + return False, "Severity property not found on Bug type" + + prop_type = getattr(severity, "property_type", None) + prop_type_val = prop_type.value if isinstance(prop_type, PropertyType) else prop_type + if str(prop_type_val or "").upper() != PropertyType.OPTION.value: + return False, f"Severity property_type={prop_type_val!r} (want OPTION)" + + option_names = { + (getattr(o, "name", None) or (o.get("name") if isinstance(o, dict) else "") or "").strip() + for o in (getattr(severity, "options", None) or []) + } + if not option_names: + try: + opts = plane.work_item_properties.options.list( + workspace_slug=workspace_slug, + project_id=project_id, + property_id=severity.id, + ) + option_names = {(getattr(o, "name", None) or "").strip() for o in (opts or [])} + except HttpError as exc: + if not is_not_found(exc): + raise + option_names = set() + + required = {"critical", "major", "minor"} + have = {n.casefold() for n in option_names if n} + missing = required - have + if missing: + return False, f"Severity options missing {sorted(missing)}; have {sorted(option_names)}" + return True, "Severity OPTION with Critical/Major/Minor present on Bug type" + + +S1_TASK: dict[str, Any] = { + "id": "S1", + "tags": {"setup", "tier1"}, + "prompt": ( + "In project {project}, add a Severity dropdown property (options: Critical, " + "Major, Minor) to the Bug work item type." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_projects", + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "create_work_item_property_option", + "retrieve_work_item_type", + "list_work_item_properties", + "retrieve_work_item_property", + "manage_work_item_type_properties", + "create_work_item_type", + "import_work_item_types_to_project", + "update_project_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S1 needs work-item-type/property schema tools " + "(resolve_work_item_type, create_work_item_property) which are " + "not on the default v2 surface — use --surface v2-schema" + ), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": { + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "list_work_item_properties", + "add_property_option", + "search_projects", + "get_workspace_context", + "get_features", + "configure_features", + "update_work_item_type", + }, + }, + }, + "needs": {"bug_type"}, + "verify": verify_s1, +} + + +async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S2: Fibonacci estimate scale exists and target item estimate_point is 5.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve active estimate + points. + try: + est = plane.estimates.retrieve(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + return False, f"no project estimate: {exc}" + est_id = getattr(est, "id", None) or as_id(est) + points = plane.estimates.list_points(workspace_slug=workspace_slug, project_id=project_id, estimate_id=est_id) + point_rows = points if isinstance(points, list) else (points.results if hasattr(points, "results") else points) + values = {(getattr(p, "value", None) or "").strip() for p in (point_rows or [])} + fib_like = {"1", "2", "3", "5", "8"} + if not fib_like.issubset(values): + ok = False + notes.append(f"estimate points missing fib subset; have {sorted(values)}") + else: + notes.append("fibonacci points present") + + five = next((p for p in (point_rows or []) if (getattr(p, "value", None) or "").strip() == "5"), None) + item = find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + ok = False + notes.append(f"target item {W8_TITLE!r} missing") + else: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + ep = getattr(detail, "estimate_point", None) + ep_id = as_id(ep) if not isinstance(ep, (int, float)) else None + # estimate_point may be expanded object or UUID. + if five is not None and ep_id and str(ep_id) == str(five.id): + notes.append("item estimate_point=5") + elif ep is not None and str(getattr(ep, "value", ep)) in ("5", "5.0"): + notes.append("item estimate value=5") + else: + ok = False + notes.append(f"item estimate_point={ep!r} (want 5)") + return ok, "; ".join(notes) + + +S2_TASK: dict[str, Any] = { + "id": "S2", + "tags": {"setup", "tier1"}, + "prompt": ( + f"In project {{project}}, add a Fibonacci estimate scale (points 1,2,3,5,8) " + f"and set the work item '{W8_TITLE}' to 5 points." + ), + "optimal_calls": 5, + "optimal_tools": { + "list_projects", + "create_project_estimate", + "create_project_estimate_points", + "link_estimate_to_project", + "update_work_item", + }, + "alternate_tools": { + "get_project_estimate", + "list_project_estimate_points", + "list_work_items", + "search_work_items", + "update_project_estimate", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S2 needs configure_estimate (schema tier) to create the Fibonacci " + "scale — default v2 has no estimate schema tools. " + "(v2 update_work_item does accept estimate_point.) Use v2-schema." + ), + }, + "v2-schema": { + # configure_estimate creates scale+points+link in one call; + # update_work_item(estimate_point="5") resolves the value server-side. + "optimal_calls": 2, + "optimal_tools": {"configure_estimate", "update_work_item"}, + "alternate_tools": { + "search_projects", + "get_features", + "find_work_items", + "get_work_item", + "get_workspace_context", + "bulk_update_work_items", + }, + }, + }, + "needs": {"items"}, + "verify": verify_s2, +} + + +async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S3: Incident type exists with a required TEXT property. + + Workspace-owned types: probe get_features.is_work_item_types_enabled at verify + time (S3 needs is empty, so seed never sets bug_type_workspace_level). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # Find Incident type (project list first). + types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) + incident = next((t for t in types if (t.name or "").strip().casefold() == "incident"), None) + if incident is None: + # Probe workspace feature at verify time — do not rely on seed ctx flags. + workspace_owns = False + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + workspace_owns = bool(dump.get("is_work_item_types_enabled")) + except Exception: + workspace_owns = False + if workspace_owns: + try: + wtypes = list(plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []) + incident = next((t for t in wtypes if (t.name or "").strip().casefold() == "incident"), None) + except Exception: + pass + if incident is None: + return False, "Incident work item type not found" + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(incident.id), + ) + or [] + ) + except HttpError as exc: + if is_not_found(exc): + return False, "no properties on Incident type" + raise + + required_text = None + for p in props: + prop_type = getattr(p, "property_type", None) + prop_type_val = str(prop_type.value if isinstance(prop_type, PropertyType) else prop_type or "").upper() + is_required = bool(getattr(p, "is_required", False)) + # TEXT only — no fallback to other required types (OPTION etc.). + if is_required and prop_type_val in (PropertyType.TEXT.value, "TEXT", "STRING"): + required_text = p + break + if required_text is None: + return False, f"no required TEXT property on Incident; props={len(props)}" + display = getattr(required_text, "display_name", None) or getattr(required_text, "name", None) + return True, f"Incident type + required TEXT property {display!r}" + + +S3_TASK: dict[str, Any] = { + "id": "S3", + "tags": {"setup", "tier1"}, + "prompt": ( + "In project {project}, create a work item type named 'Incident' and add a " + "required text property (e.g. 'Impact summary') on it." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_projects", + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "create_work_item_type", + "list_work_item_types", + "import_work_item_types_to_project", + "list_work_item_properties", + "manage_work_item_type_properties", + "update_project_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S3 needs resolve_work_item_type + create_work_item_property (schema tier) — use --surface v2-schema" + ), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": { + "resolve_work_item_type", + "create_work_item_property", + }, + "alternate_tools": { + "list_work_item_types", + "list_work_item_properties", + "update_work_item_type", + "search_projects", + "get_features", + "configure_features", + }, + }, + }, + "needs": set(), + "verify": verify_s3, +} + + +async def verify_s4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S4: billing intake accepted (status=1), spam declined (status=-1).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + intake = ctx.get("intake") or {} + billing = intake.get("billing") or {} + spam = intake.get("spam") or {} + notes: list[str] = [] + ok = True + + def _status_of(issue_id: str | None, title: str) -> int | None: + if not issue_id: + return None + try: + row = plane.intake.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=issue_id) + return getattr(row, "status", None) + except Exception: + # Fall back to list + match title + try: + rows = plane.intake.list(workspace_slug=workspace_slug, project_id=project_id) + results = rows.results if hasattr(rows, "results") else rows + for r in results or []: + detail = getattr(r, "issue_detail", None) + name = getattr(detail, "name", None) if detail is not None else None + if name and name.strip() == title: + return getattr(r, "status", None) + except Exception: + return None + return None + + b_status = _status_of(billing.get("issue_id"), INTAKE_BILLING_TITLE) + s_status = _status_of(spam.get("issue_id"), INTAKE_SPAM_TITLE) + # accept=1, decline=-1 per IntakeWorkItemStatusEnum + if b_status != 1: + ok = False + notes.append(f"billing status={b_status!r} (want 1/accepted)") + else: + notes.append("billing accepted") + if s_status != -1: + ok = False + notes.append(f"spam status={s_status!r} (want -1/declined)") + else: + notes.append("spam declined") + return ok, "; ".join(notes) + + +S4_TASK: dict[str, Any] = { + "id": "S4", + "tags": {"setup", "tier1"}, + "prompt": ( + f"In project {{project}}, triage intake: accept the billing request " + f"'{INTAKE_BILLING_TITLE}' and reject/decline the spam item " + f"'{INTAKE_SPAM_TITLE}'." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_intake_work_items", + "update_intake_work_item", + }, + "alternate_tools": { + "retrieve_intake_work_item", + "list_work_items", + "list_projects", + "create_intake_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"list_intake", "triage_intake"}, + "alternate_tools": { + "find_work_items", + "get_work_item", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"intake"}, + "verify": verify_s4, +} + + +async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S5: project cycles + worklogs AND workspace customers enabled. + + Gates (plane-ee): + - project.cycle_view — cycles create/list + - project.is_time_tracking_enabled — worklogs + - WorkspaceFeature.is_customer_enabled (API field ``customers``) — customer create 403 + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + proj = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) + cycle_view = bool(getattr(proj, "cycle_view", None)) + time_tracking = bool(getattr(proj, "is_time_tracking_enabled", None)) + if not cycle_view: + ok = False + notes.append(f"cycle_view={getattr(proj, 'cycle_view', None)!r} (want True)") + else: + notes.append("cycle_view=True") + if not time_tracking: + ok = False + notes.append(f"is_time_tracking_enabled={getattr(proj, 'is_time_tracking_enabled', None)!r} (want True)") + else: + notes.append("is_time_tracking_enabled=True") + + try: + feat = plane.projects.get_features(workspace_slug=workspace_slug, project_id=project_id) + dump = feat.model_dump() if hasattr(feat, "model_dump") else (feat if isinstance(feat, dict) else {}) + cycles_flag = dump.get("cycles") if isinstance(dump, dict) else getattr(feat, "cycles", None) + if not cycles_flag: + ok = False + notes.append(f"features.cycles={cycles_flag!r} (want True)") + else: + notes.append("features.cycles=True") + except Exception as exc: + ok = False + notes.append(f"project get_features failed: {exc}") + + # Workspace customers toggle (is_customer_enabled behind API field ``customers``). + try: + ws_feat = plane.workspaces.get_features(workspace_slug=workspace_slug) + ws_dump = ( + ws_feat.model_dump() if hasattr(ws_feat, "model_dump") else (ws_feat if isinstance(ws_feat, dict) else {}) + ) + customers_on = None + if isinstance(ws_dump, dict): + customers_on = ws_dump.get("customers") + if customers_on is None: + customers_on = ws_dump.get("is_customer_enabled") + if customers_on is None: + customers_on = getattr(ws_feat, "customers", None) + if customers_on is None: + customers_on = getattr(ws_feat, "is_customer_enabled", None) + if not customers_on: + ok = False + notes.append(f"workspace.customers={customers_on!r} (want True)") + else: + notes.append("workspace.customers=True") + except Exception as exc: + ok = False + notes.append(f"workspace get_features failed: {exc}") + + return ok, "; ".join(notes) + + +S5_TASK: dict[str, Any] = { + "id": "S5", + "tags": {"setup", "tier1"}, + "prompt": ( + "Enable cycles and time tracking (worklogs) for project {project}, " + "and enable the customers feature for the workspace." + ), + # Minimal legacy path (2 calls): + # 1. update_project(cycle_view=True, is_time_tracking_enabled=True) + # 2. update_workspace_features(customers=True) + # (features PATCH can set cycles→cycle_view but cannot set worklogs.) + "optimal_calls": 2, + "optimal_tools": {"update_project", "update_workspace_features"}, + "alternate_tools": { + "update_project_features", + "list_projects", + "retrieve_project", + "get_features", + }, + "surface_tools": { + "v2": { + "unsupported": True, + "reason": ( + "S5 needs configure_features (schema tier) for project cycles/worklogs " + "and workspace customers — use --surface v2-schema" + ), + }, + "v2-schema": { + # 2 calls: configure_features(project, cycles+worklogs) + + # configure_features(customers=True) without project. + "optimal_calls": 2, + "optimal_tools": {"configure_features"}, + "alternate_tools": { + "get_features", + "search_projects", + "get_workspace_context", + "update_work_item", + }, + }, + }, + # Seed leaves project cycles+worklogs and workspace customers off. + "needs": {"leave_cycles_worklogs_off"}, + "verify": verify_s5, +} + + +SCHEMA_TASKS: list[dict[str, Any]] = [S1_TASK, S2_TASK, S3_TASK, S4_TASK, S5_TASK] + + +__all__ = ["SCHEMA_TASKS", "verify_s1", "verify_s2", "verify_s3", "verify_s4", "verify_s5"] diff --git a/evals/tasks/write.py b/evals/tasks/write.py new file mode 100644 index 00000000..00c676ba --- /dev/null +++ b/evals/tasks/write.py @@ -0,0 +1,771 @@ +"""Write-task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.query_params import RetrieveQueryParams, WorkItemQueryParams + +from evals.seed import ( + CYCLE_CURRENT, + CYCLE_PAST, + MODULE_COMPLETED_TITLES, + MODULE_NAME, + W2_TITLE, + W3_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, + W7_URL, + W8_TITLE, +) +from evals.tasks.common import ( + find_item_by_name, + find_items_by_name, + ids, + is_not_found, + state_group, + state_name, + word_boundary, +) + + +async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W1: assert end-state via Plane API (title, priority, assignee, auth label).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + title = "Login page 500s on empty password" + matches = find_items_by_name(plane, workspace_slug, project_id, title) + if not matches: + return False, f"work item {title!r} not found" + item = matches[0] # newest first + notes: list[str] = [] + if len(matches) > 1: + notes.append(f"warning: {len(matches)} items with title (verifying newest)") + + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + params=RetrieveQueryParams(expand="assignees,labels"), + ) + + ok = True + + priority = (detail.priority or "").lower() if detail.priority else "" + if priority != "urgent": + ok = False + notes.append(f"priority={priority!r} (want urgent)") + else: + notes.append("priority=urgent") + + me = plane.users.get_me() + me_id = str(me.id) + assignee_ids = ids(detail.assignees) + if me_id not in assignee_ids: + ok = False + notes.append(f"assignees={sorted(assignee_ids)} missing me={me_id}") + else: + notes.append("assigned to me") + + auth_label_id = (ctx.get("labels") or {}).get("auth") + label_ids = ids(detail.labels) + if not auth_label_id: + ok = False + notes.append("auth label id missing from seed ctx") + elif str(auth_label_id) not in label_ids: + ok = False + notes.append(f"labels={sorted(label_ids)} missing auth={auth_label_id}") + else: + notes.append("auth label attached") + + return ok, "; ".join(notes) + + +W1_TASK: dict[str, Any] = { + "id": "W1", + "tags": {"write", "tier1"}, + "prompt": ( + "Create a work item in project {project}: title 'Login page 500s on empty " + "password', priority urgent, assign it to me, and add the 'auth' label." + ), + "optimal_calls": 4, + "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, + "alternate_tools": { + "search_work_items", + "list_states", + "retrieve_project", + "get_workspace_members", + "manage_work_item_assignee", + "manage_work_item_label", + "update_work_item", + "list_work_items", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"get_workspace_context", "create_work_item"}, + "alternate_tools": { + "search_projects", + "list_labels", + "find_work_items", + "get_work_item", + "update_work_item", + }, + }, + }, + "needs": {"labels"}, + "verify": verify_w1, +} + + +async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W2: target item is in a completed-group state (prefer name Done).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W2_TITLE) + if item is None: + return False, f"item {W2_TITLE!r} not found" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + name = state_name(plane, workspace_slug, project_id, detail.state) + group = state_group(plane, workspace_slug, project_id, detail.state) + if group == "completed" or (name and name.casefold() == "done"): + return True, f"state={name!r} group={group!r}" + return False, f"state={name!r} group={group!r} (want completed/Done)" + + +W2_TASK: dict[str, Any] = { + "id": "W2", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, move the work item titled '{W2_TITLE}' to the Done state."), + "optimal_calls": 3, + "optimal_tools": {"list_work_items", "list_states", "update_work_item"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "retrieve_state", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "update_work_item"}, + "alternate_tools": { + "list_states", + "get_work_item", + "list_available_transitions", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w2, +} + + +async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W3: target item has a comment containing the prompt phrase 'contrast tokens'.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W3_TITLE) + if item is None: + return False, f"item {W3_TITLE!r} not found" + resp = plane.work_items.comments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + results = list(resp.results if hasattr(resp, "results") else resp or []) + if not results: + return False, "no comments on target item" + phrase = "contrast tokens" + pat = word_boundary(phrase) + for c in results: + html = getattr(c, "comment_html", None) or "" + stripped = getattr(c, "comment_stripped", None) or "" + # Some APIs expose plain text under comment_stripped; fall back to html. + blob = f"{stripped}\n{html}" + if pat.search(blob): + return True, f"comment matches {phrase!r}" + return False, f"no comment contains {phrase!r} ({len(results)} comment(s))" + + +W3_TASK: dict[str, Any] = { + "id": "W3", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' " + "saying 'Reviewed contrast tokens — needs design pass'." + ), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "create_work_item_comment"}, + "alternate_tools": { + "search_work_items", + "retrieve_work_item", + "list_work_item_comments", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "create_comment"}, + "alternate_tools": { + "get_work_item", + "modify_comment", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w3, +} + + +async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W4: the seeded triage label id is now named needs-triage. + + Authoritative path: retrieve ctx['labels']['triage'] by id. Name-scan is + only a fallback when the seed id is missing from ctx. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + triage_id = (ctx.get("labels") or {}).get("triage") + if triage_id: + try: + lb = plane.labels.retrieve(workspace_slug=workspace_slug, project_id=project_id, label_id=triage_id) + name = (lb.name or "").strip().casefold() + if name in ("needs-triage", "needs triage"): + return True, f"label id {triage_id} now named {lb.name!r}" + return False, f"label id {triage_id} still named {lb.name!r}" + except HttpError as exc: + if not is_not_found(exc): + raise + return False, f"seeded triage label id {triage_id} not found (deleted?)" + + # Fallback only when seed id is absent from ctx. + page = plane.labels.list(workspace_slug=workspace_slug, project_id=project_id) + names = {(lb.name or "").strip().casefold(): (lb.name or "").strip() for lb in (page.results or [])} + if "needs-triage" in names or "needs triage" in names: + if "triage" in names: + return False, "both triage and needs-triage still present" + return True, "label renamed to needs-triage (no seed id; name-scan fallback)" + return False, f"needs-triage not found; labels={sorted(names.values())}" + + +W4_TASK: dict[str, Any] = { + "id": "W4", + "tags": {"write", "tier1"}, + "prompt": ("In project {project}, rename the label 'triage' to 'needs-triage'."), + "optimal_calls": 2, + "optimal_tools": {"list_labels", "update_label"}, + "alternate_tools": { + "retrieve_label", + "create_label", + "delete_label", + "list_projects", + }, + "surface_tools": { + "v2": { + # Default v2 has list_labels but no update_label (schema tier). + "unsupported": True, + "reason": ("W4 needs update_label which is only on the v2-schema surface — use --surface v2-schema"), + }, + "v2-schema": { + "optimal_calls": 2, + "optimal_tools": {"list_labels", "update_label"}, + "alternate_tools": { + "create_label", + "delete_label", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": {"labels"}, + "verify": verify_w4, +} + + +async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W5: all seeded module completed items are archived (not merely deleted).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + ids = [str(i) for i in (ctx.get("module_completed_ids") or [])] + if not ids: + # Fall back to titles. + for title in MODULE_COMPLETED_TITLES: + item = find_item_by_name(plane, workspace_slug, project_id, title) + if item: + ids.append(str(item.id)) + if not ids: + return False, "no module completed item ids" + + not_archived: list[str] = [] + need_archive_list: list[str] = [] # 404 on retrieve — must appear in archived list + for wid in ids: + try: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except HttpError as exc: + if is_not_found(exc): + # Deleted OR archived-as-404 — require confirmation via list_archived. + need_archive_list.append(str(wid)) + continue + raise + archived_at = getattr(detail, "archived_at", None) + if not archived_at: + not_archived.append(str(wid)) + + arch_ids: set[str] = set() + if need_archive_list or not_archived: + try: + arch = plane.work_items.list_archived( + workspace_slug=workspace_slug, + project_id=project_id, + params=WorkItemQueryParams(per_page=100), + ) + arch_ids = {str(i.id) for i in (arch.results or [])} + except Exception as exc: + if need_archive_list: + return False, f"list_archived failed while confirming 404 items: {exc}" + + # 404s only count as archived if present on the archived list (deletes fail). + for wid in need_archive_list: + if wid not in arch_ids: + not_archived.append(wid) + not_archived = [i for i in not_archived if i not in arch_ids] + + if not_archived: + return False, f"{len(not_archived)} module items not archived: {not_archived}" + return True, f"{len(ids)} module completed items archived" + + +W5_TASK: dict[str, Any] = { + "id": "W5", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, archive all completed work items in the module '{MODULE_NAME}'."), + "optimal_calls": 5, # list_modules + list_module_work_items + 3× archive + "optimal_tools": { + "list_modules", + "list_module_work_items", + "manage_work_item_archive", + }, + "alternate_tools": { + "list_work_items", + "retrieve_module", + "list_projects", + "list_states", + }, + "surface_tools": { + "v2": { + "optimal_calls": 5, + "optimal_tools": {"list_modules", "find_work_items", "archive_work_item"}, + "alternate_tools": { + "get_work_item", + "assign_to_module", + "search_projects", + "list_states", + }, + }, + }, + "needs": {"module"}, + "verify": verify_w5, +} + + +async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W6: Sprint 12 closed by a real completion signal + unfinished items on Sprint 13. + + complete_cycle (SDK) sets end_date to *today* — a no-op agent leaves the seeded + past end_date unchanged, so requiring end_date==today (or archived_at set) is + non-vacuous. progress_snapshot non-null is also accepted when the API flips it. + """ + from datetime import date as _date + + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + past_id = ctx.get("cycle_past_id") or (ctx.get("cycles") or {}).get(CYCLE_PAST) + cur_id = ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) + if not past_id: + return False, "Sprint 12 id missing from seed" + notes: list[str] = [] + ok = True + past = plane.cycles.retrieve(workspace_slug=workspace_slug, project_id=project_id, cycle_id=past_id) + end = getattr(past, "end_date", None) + archived_at = getattr(past, "archived_at", None) + snapshot = getattr(past, "progress_snapshot", None) + today = _date.today().isoformat() + seed_end = ctx.get("cycle_past_seed_end_date") + + # Real close signals (any one suffices): + # 1) complete_cycle → end_date becomes today + # 2) manage_cycle_archive → archived_at set + # 3) progress_snapshot populated (Plane completion snapshot) + # end_date comes back as a timestamp ('2026-08-12T00:00:00Z'), so compare the + # date part — a whole-string match against today's date can never be true. + end_day = str(end or "")[:10] + closed = False + if archived_at: + closed = True + notes.append(f"Sprint 12 archived_at={archived_at}") + elif end_day == today: + closed = True + notes.append(f"Sprint 12 end_date={end} (complete_cycle today)") + elif snapshot not in (None, {}, []): + closed = True + notes.append("Sprint 12 progress_snapshot set") + if not closed: + ok = False + notes.append( + f"Sprint 12 not closed: end_date={end!r} seed_end={seed_end!r} " + f"archived_at={archived_at!r} snapshot={snapshot!r} " + f"(want end_date={today!r} or archived_at or progress_snapshot)" + ) + + unfinished = list(ctx.get("w6_unfinished_titles") or []) + if cur_id and unfinished: + try: + on13 = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=cur_id, + params=WorkItemQueryParams(per_page=100), + ) + names = {(i.name or "").strip() for i in (on13.results or [])} + missing = [t for t in unfinished if t not in names] + if missing: + ok = False + notes.append(f"unfinished not on Sprint 13: {missing}") + else: + notes.append(f"{len(unfinished)} unfinished on Sprint 13") + except Exception as exc: + notes.append(f"list Sprint 13 items failed: {exc}") + return ok, "; ".join(notes) + + +W6_TASK: dict[str, Any] = { + "id": "W6", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, '{CYCLE_PAST}' is wrapping up. Close it and make sure " + f"its unfinished work items end up on '{CYCLE_CURRENT}'." + ), + "optimal_calls": 4, + "optimal_tools": { + "list_cycles", + "transfer_cycle_work_items", + "complete_cycle", + }, + "alternate_tools": { + "list_cycle_work_items", + "manage_cycle_work_items", + "update_cycle", + "list_work_items", + "list_projects", + }, + "surface_tools": { + "v2": { + # close_cycle with transfer_to is the consolidated path. + "optimal_calls": 2, + "optimal_tools": {"list_cycles", "close_cycle"}, + "alternate_tools": { + "assign_to_cycle", + "find_work_items", + "search_projects", + "get_workspace_context", + }, + }, + }, + # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — + # Plane rejects every edit to an ended cycle. See _seed_cycles. + "needs": {"items", "cycles", "cycles_open_past"}, + "verify": verify_w6, +} + + +async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W7: source blocks target (dependency) AND reference URL link exists on source. + + Only dump['blocking'] ids count — a reverse blocked_by match must not pass. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + src = find_item_by_name(plane, workspace_slug, project_id, W7_SOURCE_TITLE) + tgt = find_item_by_name(plane, workspace_slug, project_id, W7_TARGET_TITLE) + if not src or not tgt: + return False, "W7 source/target items not found" + notes: list[str] = [] + ok = True + + # Dependencies — require tgt in blocking specifically. + try: + deps = plane.work_items.dependencies.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + dump = deps.model_dump() if hasattr(deps, "model_dump") else (deps if isinstance(deps, dict) else {}) + blocking = dump.get("blocking") or [] + if isinstance(blocking, dict): + blocking = blocking.get("results") or list(blocking.values()) + blocking_ids = ids(blocking) + # blocking may also be plain UUID strings + for b in blocking if isinstance(blocking, list) else []: + if isinstance(b, str): + blocking_ids.add(b) + if str(tgt.id) not in blocking_ids: + ok = False + blob_hit = str(tgt.id) in str(dump) + note = f"no blocking relation from source to {tgt.id}; blocking_ids={sorted(blocking_ids)}" + if blob_hit: + note += " (target id appears elsewhere in dump — wrong direction)" + notes.append(note) + else: + notes.append("blocking relation present") + except Exception as exc: + ok = False + notes.append(f"dependencies list failed: {exc}") + + # Links + try: + links = plane.work_items.links.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + rows = links.results if hasattr(links, "results") else links + urls = {(getattr(ln, "url", None) or "").strip() for ln in (rows or [])} + if W7_URL not in urls: + ok = False + notes.append(f"link {W7_URL!r} missing; have {sorted(urls)}") + else: + notes.append("reference URL present") + except Exception as exc: + ok = False + notes.append(f"links list failed: {exc}") + + return ok, "; ".join(notes) + + +W7_TASK: dict[str, Any] = { + "id": "W7", + "tags": {"write", "tier1"}, + "prompt": ( + f"In project {{project}}, mark the work item '{W7_SOURCE_TITLE}' as blocking " + f"'{W7_TARGET_TITLE}', and add the reference URL {W7_URL} on the blocking item." + ), + "optimal_calls": 3, + "optimal_tools": { + "list_work_items", + "create_work_item_relation", + "create_work_item_link", + }, + "alternate_tools": { + "search_work_items", + "list_work_item_relations", + "list_work_item_relation_definitions", + "list_work_item_links", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 3, + "optimal_tools": {"find_work_items", "link_work_items", "add_work_item_link"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "update_work_item", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w7, +} + + +async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W8: work log of exactly 120 minutes exists on the target item. + + Note: plane-sdk Create work log has no logged-date field — 'yesterday' in the + prompt cannot be asserted; only duration is verified. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + return False, f"item {W8_TITLE!r} not found" + logs = plane.work_items.work_logs.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 120 in durations: + return True, "work log duration=120 present" + return False, f"no 120-minute work log; durations={durations}" + + +W8_TASK: dict[str, Any] = { + "id": "W8", + "tags": {"write", "tier1"}, + "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}' for yesterday."), + "optimal_calls": 2, + "optimal_tools": {"list_work_items", "create_work_log"}, + "alternate_tools": { + "search_work_items", + "list_work_logs", + "retrieve_work_item", + "list_projects", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "log_work"}, + "alternate_tools": { + "get_work_item", + "search_projects", + "update_work_item", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w8, +} + + +async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W9 (extra): bulk priority change — the three non-R1 urgent titles are now high.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # All urgent fixtures except we ask agent to set medium-priority batch targets. + # Prompt targets the three titles starting with Session/Inventory/Checkout (non-R1 urgent). + targets = [ + "Checkout times out on 3DS challenge", + "Session cookie not rotated after login", + "Inventory count goes negative under load", + ] + wrong: list[str] = [] + for title in targets: + item = find_item_by_name(plane, workspace_slug, project_id, title) + if not item: + wrong.append(f"{title}: missing") + continue + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + pr = (detail.priority or "").lower() + if pr != "high": + wrong.append(f"{title}: priority={pr!r}") + if wrong: + return False, "; ".join(wrong) + return True, "3 items priority=high" + + +W9_TASK: dict[str, Any] = { + "id": "W9", + "tags": {"write", "tier1", "extra"}, + "prompt": ( + "In project {project}, set priority to high on these three work items in one " + "batch: 'Checkout times out on 3DS challenge', " + "'Session cookie not rotated after login', " + "'Inventory count goes negative under load'." + ), + # Extra: exercises bulk_update_work_items (not in original DESIGN 20). + "optimal_calls": 4, + "optimal_tools": { + "list_work_items", + "update_work_item", + }, + "alternate_tools": { + "search_work_items", + "list_projects", + "retrieve_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 2, + "optimal_tools": {"find_work_items", "bulk_update_work_items"}, + "alternate_tools": { + "update_work_item", + "get_work_item", + "search_projects", + }, + }, + }, + "needs": {"items"}, + "verify": verify_w9, +} + + +async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W10 (extra): project page named Eval Runbook exists.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + try: + resp = plane.pages.list_project_pages(workspace_slug=workspace_slug, project_id=project_id) + rows = resp.results if hasattr(resp, "results") else resp + except Exception as exc: + return False, f"list pages failed: {exc}" + names = {(getattr(p, "name", None) or "").strip() for p in (rows or [])} + if "Eval Runbook" not in names: + return False, f"page 'Eval Runbook' missing; have {sorted(names)}" + return True, "page Eval Runbook present" + + +W10_TASK: dict[str, Any] = { + "id": "W10", + "tags": {"write", "tier1", "extra"}, + "prompt": ( + "In project {project}, create a project page named 'Eval Runbook' with body " + "text 'Rollback steps for eval harness'." + ), + # Extra: exercises pages family (create_page / get_page). + "optimal_calls": 2, + "optimal_tools": {"list_projects", "create_page"}, + "alternate_tools": { + "list_pages", + "retrieve_page", + "attach_page_to_work_item", + }, + "surface_tools": { + "v2": { + "optimal_calls": 1, + "optimal_tools": {"create_page"}, + "alternate_tools": { + "list_pages", + "get_page", + "search_projects", + "get_workspace_context", + }, + }, + }, + "needs": set(), + "verify": verify_w10, +} + + +WRITE_TASKS: list[dict[str, Any]] = [ + W1_TASK, + W2_TASK, + W3_TASK, + W4_TASK, + W5_TASK, + W6_TASK, + W7_TASK, + W8_TASK, + W9_TASK, + W10_TASK, +] + + +__all__ = [ + "WRITE_TASKS", + "verify_w1", + "verify_w2", + "verify_w3", + "verify_w4", + "verify_w5", + "verify_w6", + "verify_w7", + "verify_w8", + "verify_w9", + "verify_w10", +] diff --git a/tests/test_evals_catalog.py b/tests/test_evals_catalog.py index 45cb1b12..23faf2ee 100644 --- a/tests/test_evals_catalog.py +++ b/tests/test_evals_catalog.py @@ -41,6 +41,42 @@ LONG_TAIL_IDS = {"L1", "L2", "L3", "L4", "L5"} # Workspace-scoped prompts that omit {project} NO_PROJECT_PROMPT_IDS = {"C2", "L3", "L4"} +CATALOG_ID_ORDER = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) @pytest.fixture(autouse=True) @@ -61,6 +97,10 @@ def test_catalog_includes_design_and_extras(): assert len(TASKS) >= 20 +def test_catalog_id_order_is_pinned(): + assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER + + def test_get_tasks_all_and_filter(): all_t = get_tasks(None) assert len(all_t) == len(TASKS) @@ -238,11 +278,19 @@ def test_seed_plan_empty_needs_only_project(): def test_verifiers_are_async_and_importable(): + modules = { + "R": "read", + "W": "write", + "S": "schema", + "C": "cross", + "I": "debias", + "L": "debias", + } for t in TASKS: fn = t["verify"] assert inspect.iscoroutinefunction(fn), t["id"] # Callables resolve without NameError - assert fn.__module__ == "evals.tasks" + assert fn.__module__ == f"evals.tasks.{modules[t['id'][0]]}" def test_cmd_list_prints_all_task_ids(capsys): From 19cd92fb204204f58352adb40c7f095e4152a4f9 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 02:02:43 +0530 Subject: [PATCH 06/93] Separate the CLI from the run loop run.py was argparse wiring, model-alias resolution, the live run loop and the canary in one 976-line module, so the entry point and the machinery it drives could not be read or tested apart. cli.py now owns argument parsing and dispatch, runner.py owns run_live, the canary and their row/resume/meta bookkeeping, and runner never imports argparse. run.py stays as a thin delegating shim because python -m evals.run appears in every script, runbook and doc we have; its CLI behaviour and --help are unchanged. Co-Authored-By: Claude Fable 5 --- evals/cli.py | 284 +++++++++ evals/run.py | 1019 +++------------------------------ evals/runner.py | 746 ++++++++++++++++++++++++ tests/test_evals_hardening.py | 67 +-- tests/test_evals_proxy.py | 2 +- tests/test_evals_surface.py | 2 +- 6 files changed, 1133 insertions(+), 987 deletions(-) create mode 100644 evals/cli.py create mode 100644 evals/runner.py diff --git a/evals/cli.py b/evals/cli.py new file mode 100644 index 00000000..a9a69e1c --- /dev/null +++ b/evals/cli.py @@ -0,0 +1,284 @@ +"""Command-line wiring and model resolution for the eval harness. + +The stable process entry point remains ``python -m evals.run``; that module +delegates here. +""" + +from __future__ import annotations + +import argparse +import asyncio +import shlex +import sys +import uuid +from pathlib import Path +from typing import Any + +from evals.drivers import KNOWN_DRIVERS +from evals.runner import KNOWN_SURFACES, run_canary, run_live +from evals.seed import seed_plan +from evals.tasks import TASKS, format_task_prompt, get_tasks + +MODEL_ALIASES: dict[str, str] = { + "sonnet": "claude-sonnet-5", + "haiku": "claude-haiku-4-5", +} +API_MODEL_ALIASES: dict[str, dict[str, str]] = { + "anthropic": MODEL_ALIASES, + # Preserve the harness's representative/fast intent when the user switches + # providers without also overriding the historical sonnet/haiku aliases. + "openai": {"sonnet": "gpt-5", "haiku": "gpt-5-mini"}, +} +# Per-driver resolution of the short harness aliases (sonnet/haiku). +# Drivers that need provider/model form get qualified defaults; unknown +# strings (e.g. ``anthropic/claude-…``) pass through unchanged. +CLI_MODEL_ALIASES: dict[str, dict[str, str]] = { + "claude-cli": {"sonnet": "sonnet", "haiku": "haiku"}, + "codex-cli": {"sonnet": "sonnet", "haiku": "haiku"}, + "antigravity-cli": { + "sonnet": "gemini-3.6-flash-high", + "haiku": "gemini-3.6-flash-low", + }, + "opencode-cli": { + "sonnet": "anthropic/claude-sonnet-4-20250514", + "haiku": "anthropic/claude-haiku-4-5-20251001", + }, +} + +DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "results" + + +def resolve_model_for_driver(driver_name: str, model: str, *, provider: str = "anthropic") -> str: + """Map a harness model token to the string the given driver expects. + + Known short aliases (sonnet/haiku) are looked up per-driver. Any other + string (including already-qualified ``provider/model``) is passed through. + """ + key = (driver_name or "api").strip().lower() + if key in ("api", "sdk"): + table = API_MODEL_ALIASES.get(provider.strip().lower()) or {} + return table.get(model, model) + table = CLI_MODEL_ALIASES.get(key) or {} + return table.get(model, model) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Plane MCP tool-surface eval harness") + p.add_argument("--list", action="store_true", help="Print task table (no network)") + p.add_argument("--dry-run", action="store_true", help="Print resolved prompts + seed plan (no network)") + p.add_argument("--tasks", type=str, default=None, help="Comma-separated task ids (default: all)") + p.add_argument( + "--model", + type=str, + default="sonnet", + help=( + "Model alias (sonnet/haiku) or a free-form provider/model id. " + "Short aliases are remapped per --driver (opencode/antigravity get qualified names)." + ), + ) + p.add_argument("--reps", type=int, default=1, help="Repetitions per task") + p.add_argument( + "--surface", + type=str, + default="full", + help=( + "Tool surface: 'full' (legacy 177 tools), 'v2', or 'v2-schema'. " + "With --server-cmd it is a free-form label for the external surface." + ), + ) + p.add_argument( + "--server-cmd", + type=str, + default=None, + help=( + "External MCP stdio server launch command (shlex-split), e.g. " + "'/path/venv/bin/python -m plane_mcp stdio --v2'. Enables external mode: " + "all tasks run (no surface skips) and mispick classification is disabled " + "(the foreign tool names have no overlay sets)." + ), + ) + p.add_argument( + "--server-env", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra env var for the (external) MCP server child; repeatable.", + ) + p.add_argument( + "--driver", + type=str, + default="api", + choices=sorted(KNOWN_DRIVERS), + help=( + "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli " + "('sdk' is an alias for 'api'). Not required for --canary." + ), + ) + p.add_argument( + "--provider", + type=str, + default="anthropic", + choices=("anthropic", "openai"), + help="Model API provider for --driver api/sdk (default: anthropic).", + ) + p.add_argument( + "--record-result-payloads", + action="store_true", + help=( + "CLI drivers only: record serialized tool-result text for tokenizer counting " + "(off by default; sidecars may contain live workspace data)" + ), + ) + p.add_argument("--out", type=str, default=None, help="JSONL output path") + p.add_argument( + "--resume", + type=str, + default=None, + metavar="OUT.jsonl", + help=( + "Resume into an existing JSONL (also the --out target). Skip (task_id, rep) " + "pairs that already completed; re-run rows with infra_ error_class or non-null error." + ), + ) + p.add_argument( + "--canary", + action="store_true", + help=( + "Verifier canary: seed each task, call verify with an empty agent result " + "(no driver/model), teardown. Exit 1 if any verifier returns ok=True on do-nothing." + ), + ) + return p.parse_args(argv) + + +def _task_ids(raw: str | None) -> list[str] | None: + if raw is None: + return None + return [t.strip() for t in raw.split(",") if t.strip()] + + +def cmd_list() -> int: + print(f"{'id':<6} {'tags':<18} {'opt':>4} prompt") + print("-" * 100) + for task in TASKS: + tags = ",".join(sorted(task["tags"])) + prompt = task["prompt"].replace("\n", " ") + if len(prompt) > 70: + prompt = prompt[:67] + "..." + print(f"{task['id']:<6} {tags:<18} {task['optimal_calls']:>4} {prompt}") + return 0 + + +def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: + needs: set[str] = set() + for task in tasks: + needs |= set(task.get("needs") or set()) + print("Seed plan:") + for line in seed_plan(needs): + print(f" {line}") + print() + sample_ctx = {"project_name": "EVAL deadbeef"} + for task in tasks: + resolved = format_task_prompt(task, sample_ctx, strict=False) + print(f"=== {task['id']} ===") + print(f"needs: {sorted(task.get('needs') or [])}") + print(f"author: {task.get('author') or 'claude'}") + print(f"optimal_calls: {task['optimal_calls']}") + print(f"optimal_tools: {sorted(task['optimal_tools'])}") + print(f"prompt:\n {resolved}") + print() + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + + if args.list: + return cmd_list() + + ids = _task_ids(args.tasks) + try: + tasks = get_tasks(ids) + except SystemExit as exc: + print(exc, file=sys.stderr) + return 2 + + if args.dry_run: + return cmd_dry_run(tasks) + + surface = (args.surface or "full").strip().lower() + server_cmd: list[str] | None = None + if args.server_cmd: + server_cmd = shlex.split(args.server_cmd) + if not server_cmd: + print("error: --server-cmd is empty", file=sys.stderr) + return 2 + elif surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " + "(or pass --server-cmd for an external surface)", + file=sys.stderr, + ) + return 2 + + server_env: dict[str, str] = {} + for pair in args.server_env: + key, sep, val = pair.partition("=") + if not sep or not key: + print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) + return 2 + server_env[key] = val + + # Canary: live env only — no driver/model required. + if args.canary: + return asyncio.run(run_canary(tasks, surface=surface)) + + driver_name = (getattr(args, "driver", None) or "api").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + + if args.resume: + out = Path(args.resume) + elif args.out: + out = Path(args.out) + else: + out = DEFAULT_OUT_DIR / f"{uuid.uuid4().hex}.jsonl" + + model_id = resolve_model_for_driver(driver_name, args.model, provider=args.provider) + return asyncio.run( + run_live( + tasks, + model_alias=args.model, + reps=args.reps, + surface=surface, + out_path=out, + driver_name=driver_name, + provider=args.provider, + server_cmd=server_cmd, + server_env=server_env or None, + resume=bool(args.resume), + record_result_payloads=bool(args.record_result_payloads), + resolved_model_id=model_id, + ) + ) + + +__all__ = [ + "API_MODEL_ALIASES", + "CLI_MODEL_ALIASES", + "DEFAULT_OUT_DIR", + "MODEL_ALIASES", + "cmd_dry_run", + "cmd_list", + "main", + "parse_args", + "resolve_model_for_driver", +] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/run.py b/evals/run.py index 2ac1c4ba..c084be28 100644 --- a/evals/run.py +++ b/evals/run.py @@ -1,543 +1,43 @@ -"""CLI driver for the Plane MCP tool-surface eval harness. +"""Compatibility entry point for the Plane MCP eval harness. -Usage: - python -m evals.run --list - python -m evals.run --dry-run --tasks R1 - python -m evals.run --tasks R1,W1,S1 --model sonnet --reps 1 --surface full +CLI concerns live in :mod:`evals.cli`; live execution and result bookkeeping +live in :mod:`evals.runner`. Existing imports and ``python -m evals.run`` remain +stable through this façade. """ from __future__ import annotations -import argparse -import asyncio -import json -import os -import subprocess -import sys -import uuid -from datetime import datetime, timezone from pathlib import Path from typing import Any -from evals.drivers import ( - KNOWN_DRIVERS, - agent_run_to_harness_dict, - get_driver, +from evals import runner +from evals.cli import ( + API_MODEL_ALIASES, + CLI_MODEL_ALIASES, + DEFAULT_OUT_DIR, + MODEL_ALIASES, + cmd_dry_run, + cmd_list, + main, + parse_args, + resolve_model_for_driver, ) -from evals.seed import make_plane_client, seed, seed_plan, teardown -from evals.tasks import ( - TASKS, - PromptBindError, - TaskSkipped, - battery_fingerprint, - format_task_prompt, - get_tasks, - resolve_surface_tool_sets, - task_author, +from evals.runner import ( + KNOWN_SURFACES, + MAX_ITERATIONS, + MAX_TOKENS, + classify_call, + is_infra_cli_stop_reason, + is_meta_or_non_task_row, + load_resume_skip_keys, + make_run_meta_row, + maybe_write_run_meta, + run_agent_task_via_driver, + run_canary, + should_skip_resume_row, + stdio_server_env, ) -MODEL_ALIASES: dict[str, str] = { - "sonnet": "claude-sonnet-5", - "haiku": "claude-haiku-4-5", -} -API_MODEL_ALIASES: dict[str, dict[str, str]] = { - "anthropic": MODEL_ALIASES, - # Preserve the harness's representative/fast intent when the user switches - # providers without also overriding the historical sonnet/haiku aliases. - "openai": {"sonnet": "gpt-5", "haiku": "gpt-5-mini"}, -} -# Per-driver resolution of the short harness aliases (sonnet/haiku). -# Drivers that need provider/model form get qualified defaults; unknown -# strings (e.g. ``anthropic/claude-…``) pass through unchanged. -CLI_MODEL_ALIASES: dict[str, dict[str, str]] = { - "claude-cli": {"sonnet": "sonnet", "haiku": "haiku"}, - "codex-cli": {"sonnet": "sonnet", "haiku": "haiku"}, - "antigravity-cli": { - "sonnet": "gemini-3.6-flash-high", - "haiku": "gemini-3.6-flash-low", - }, - "opencode-cli": { - "sonnet": "anthropic/claude-sonnet-4-20250514", - "haiku": "anthropic/claude-haiku-4-5-20251001", - }, -} - - -def resolve_model_for_driver(driver_name: str, model: str, *, provider: str = "anthropic") -> str: - """Map a harness model token to the string the given driver expects. - - Known short aliases (sonnet/haiku) are looked up per-driver. Any other - string (including already-qualified ``provider/model``) is passed through. - """ - key = (driver_name or "api").strip().lower() - if key in ("api", "sdk"): - table = API_MODEL_ALIASES.get(provider.strip().lower()) or {} - return table.get(model, model) - table = CLI_MODEL_ALIASES.get(key) or {} - return table.get(model, model) - - -# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). -# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env -# (see plane_mcp.v2.choose_stdio_mcp). -KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) - -DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "results" -MAX_ITERATIONS = 15 -MAX_TOKENS = 8192 - - -def _git_sha() -> str: - try: - return ( - subprocess.check_output( - ["git", "rev-parse", "HEAD"], - stderr=subprocess.DEVNULL, - cwd=Path(__file__).resolve().parent.parent, - ) - .decode() - .strip() - ) - except Exception: - return "unknown" - - -def _system_preamble(workspace_slug: str, project_name: str) -> str: - """Keep under 100 words — part of measured context.""" - return ( - f"You are evaluating Plane project management tools. " - f"Workspace slug: {workspace_slug}. Project name: {project_name}. " - f"Complete the task using the available tools, then stop." - ) - - -def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: - if tool in optimal: - return "optimal" - if tool in alternate: - return "alternate" - return "out_of_set" - - -def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: - """Build MCP stdio env from scratch — never inherit os.environ (F6). - - ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the - v2 tool registry. ``surface=full`` leaves the var unset (legacy default). - Other surface names (external servers under benchmark) set nothing; their - selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. - """ - env: dict[str, str] = {} - if path := os.environ.get("PATH"): - env["PATH"] = path - if home := os.environ.get("HOME"): - env["HOME"] = home - env["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] - env["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] - env["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") - if surface == "v2": - env["PLANE_MCP_SURFACE"] = "v2" - elif surface == "v2-schema": - env["PLANE_MCP_SURFACE"] = "v2-schema" - if extra: - env.update(extra) - return env - - -def should_skip_resume_row(row: dict[str, Any]) -> bool: - """Return True if a prior row is a completed result that resume should skip. - - Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. - Rows with ``skipped`` set are treated as complete and are not retried (intentional: - surface/plan skips are stable outcomes, not infra failures). - Pure function — unit-tested without the live battery. - """ - ec = row.get("error_class") - if isinstance(ec, str) and ec.startswith("infra_"): - return False - if row.get("error") is not None: - return False - return True - - -def _resume_field_mismatch( - row: dict[str, Any], - *, - field: str, - expected: str | None, -) -> str | None: - """Return an error message if row[field] is present and disagrees with expected.""" - if expected is None: - return None - raw = row.get(field) - if raw is None or raw == "": - return None # back-compat: older rows without the key pass - # Surface/driver/provider compare case-insensitively; battery/model are exact. - if field in ("surface", "driver", "provider"): - got, want = str(raw).strip().lower(), expected.strip().lower() - else: - got, want = str(raw).strip(), expected.strip() - if got != want: - return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" - return None - - -def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: - """True when a CLI AgentRun stop_reason should be classified as infra_cli. - - ``timeout`` and Claude error subtypes (``error_during_execution``, bare - ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task - failure and stays in the success-rate denominator. - """ - if not stop_reason: - return False - sr = str(stop_reason) - if sr == "timeout": - return True - if sr == "error_max_turns": - return False - if sr == "error" or sr.startswith("error_"): - return True - return False - - -def _timeout_error_message(agent: dict[str, Any]) -> str: - """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" - for note in agent.get("driver_notes") or []: - if isinstance(note, str) and note.startswith("timeout after"): - return note - return "timeout" - - -def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: - """True for run-header meta lines or any row without a task_id.""" - if row.get("row_type") == "meta": - return True - return row.get("task_id") is None - - -def load_resume_skip_keys( - path: Path, - *, - surface: str, - battery: str | None = None, - model: str | None = None, - driver: str | None = None, - provider: str | None = None, -) -> tuple[set[tuple[str, int]], int, int]: - """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. - - Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` - (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / - battery / model / driver / provider disagrees with the current run (missing keys pass for - back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked - but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. - """ - if not path.is_file(): - return set(), 0, 0 - skip_keys: set[tuple[str, int]] = set() - seen: set[tuple[str, int]] = set() - with path.open(encoding="utf-8") as fh: - for line_no, line in enumerate(fh, start=1): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError as exc: - print( - f"warning: --resume {path}:{line_no}: skipping invalid JSON ({exc})", - file=sys.stderr, - ) - continue - if not isinstance(row, dict): - continue - for field, expected in ( - ("surface", surface), - ("battery", battery), - ("driver", driver), - ("provider", provider), - ): - msg = _resume_field_mismatch(row, field=field, expected=expected) - if msg: - raise SystemExit(msg) - # New API rows keep both the requested ID (resume identity) and the - # provider-reported model that actually ran. Older rows only have model. - model_row = dict(row) - if model_row.get("requested_model"): - model_row["model"] = model_row["requested_model"] - msg = _resume_field_mismatch(model_row, field="model", expected=model) - if msg: - raise SystemExit(msg) - # Meta / header rows: checked above, not part of resume key set. - if is_meta_or_non_task_row(row): - continue - tid = row.get("task_id") - rep = row.get("rep") - if tid is None or rep is None: - continue - key = (str(tid), int(rep)) - seen.add(key) - if should_skip_resume_row(row): - skip_keys.add(key) - else: - # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. - skip_keys.discard(key) - n_retry = len(seen - skip_keys) - return skip_keys, len(skip_keys), n_retry - - -def make_run_meta_row( - *, - run_id: str, - surface: str, - battery: str, - model: str | None, - driver: str, - git_sha: str, - provider: str | None = None, - ts: str | None = None, -) -> dict[str, Any]: - """Build the single first-line meta record for a new output JSONL.""" - return { - "row_type": "meta", - "run_id": run_id, - "surface": surface, - "battery": battery, - "model": model, - "driver": driver, - "provider": provider, - "git_sha": git_sha, - "ts": ts or datetime.now(timezone.utc).isoformat(), - } - - -def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: - """Write meta as the first line when the file is missing or empty. Returns True if written.""" - path.parent.mkdir(parents=True, exist_ok=True) - if path.is_file() and path.stat().st_size > 0: - return False - with path.open("w", encoding="utf-8") as fh: - fh.write(json.dumps(meta, default=str) + "\n") - return True - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - p = argparse.ArgumentParser(description="Plane MCP tool-surface eval harness") - p.add_argument("--list", action="store_true", help="Print task table (no network)") - p.add_argument("--dry-run", action="store_true", help="Print resolved prompts + seed plan (no network)") - p.add_argument("--tasks", type=str, default=None, help="Comma-separated task ids (default: all)") - p.add_argument( - "--model", - type=str, - default="sonnet", - help=( - "Model alias (sonnet/haiku) or a free-form provider/model id. " - "Short aliases are remapped per --driver (opencode/antigravity get qualified names)." - ), - ) - p.add_argument("--reps", type=int, default=1, help="Repetitions per task") - p.add_argument( - "--surface", - type=str, - default="full", - help=( - "Tool surface: 'full' (legacy 177 tools), 'v2', or 'v2-schema'. " - "With --server-cmd it is a free-form label for the external surface." - ), - ) - p.add_argument( - "--server-cmd", - type=str, - default=None, - help=( - "External MCP stdio server launch command (shlex-split), e.g. " - "'/path/venv/bin/python -m plane_mcp stdio --v2'. Enables external mode: " - "all tasks run (no surface skips) and mispick classification is disabled " - "(the foreign tool names have no overlay sets)." - ), - ) - p.add_argument( - "--server-env", - action="append", - default=[], - metavar="KEY=VAL", - help="Extra env var for the (external) MCP server child; repeatable.", - ) - p.add_argument( - "--driver", - type=str, - default="api", - choices=sorted(KNOWN_DRIVERS), - help=( - "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli " - "('sdk' is an alias for 'api'). Not required for --canary." - ), - ) - p.add_argument( - "--provider", - type=str, - default="anthropic", - choices=("anthropic", "openai"), - help="Model API provider for --driver api/sdk (default: anthropic).", - ) - p.add_argument( - "--record-result-payloads", - action="store_true", - help=( - "CLI drivers only: record serialized tool-result text for tokenizer counting " - "(off by default; sidecars may contain live workspace data)" - ), - ) - p.add_argument("--out", type=str, default=None, help="JSONL output path") - p.add_argument( - "--resume", - type=str, - default=None, - metavar="OUT.jsonl", - help=( - "Resume into an existing JSONL (also the --out target). Skip (task_id, rep) " - "pairs that already completed; re-run rows with infra_ error_class or non-null error." - ), - ) - p.add_argument( - "--canary", - action="store_true", - help=( - "Verifier canary: seed each task, call verify with an empty agent result " - "(no driver/model), teardown. Exit 1 if any verifier returns ok=True on do-nothing." - ), - ) - return p.parse_args(argv) - - -def _task_ids(raw: str | None) -> list[str] | None: - if raw is None: - return None - return [t.strip() for t in raw.split(",") if t.strip()] - - -def cmd_list() -> int: - print(f"{'id':<6} {'tags':<18} {'opt':>4} prompt") - print("-" * 100) - for t in TASKS: - tags = ",".join(sorted(t["tags"])) - prompt = t["prompt"].replace("\n", " ") - if len(prompt) > 70: - prompt = prompt[:67] + "..." - print(f"{t['id']:<6} {tags:<18} {t['optimal_calls']:>4} {prompt}") - return 0 - - -def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: - needs: set[str] = set() - for t in tasks: - needs |= set(t.get("needs") or set()) - print("Seed plan:") - for line in seed_plan(needs): - print(f" {line}") - print() - sample_ctx = {"project_name": "EVAL deadbeef"} - for t in tasks: - resolved = format_task_prompt(t, sample_ctx, strict=False) - print(f"=== {t['id']} ===") - print(f"needs: {sorted(t.get('needs') or [])}") - print(f"author: {t.get('author') or 'claude'}") - print(f"optimal_calls: {t['optimal_calls']}") - print(f"optimal_tools: {sorted(t['optimal_tools'])}") - print(f"prompt:\n {resolved}") - print() - return 0 - - -async def run_agent_task_via_driver( - *, - driver: Any, - model_id: str | None, - task: dict[str, Any], - ctx: dict[str, Any], - workspace_slug: str, - surface: str = "full", - optimal_tools: set[str] | None = None, - alternate_tools: set[str] | None = None, - server_env: dict[str, str] | None = None, -) -> dict[str, Any]: - """Run one task through the selected AgentDriver.""" - project_name = ctx["project_name"] - system = _system_preamble(workspace_slug, project_name) - prompt = format_task_prompt(task, ctx, strict=True) - optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) - alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) - assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" - - mcp_env = stdio_server_env(surface=surface, extra=server_env) - # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. - agent_run = await asyncio.to_thread( - driver.run_task, - prompt, - mcp_env, - model_id, - MAX_ITERATIONS, - system=system, - cwd=Path(__file__).resolve().parent.parent, - ) - return agent_run_to_harness_dict( - agent_run, - optimal=optimal, - alternate=alternate, - classify=classify_call, - ) - - -def _base_row( - *, - run_id: str, - git_sha: str, - surface: str, - driver_name: str, - provider: str | None, - model_id: str | None, - task: dict[str, Any], - rep: int, - battery: str, - classification: str, -) -> dict[str, Any]: - return { - "run_id": run_id, - "ts": datetime.now(timezone.utc).isoformat(), - "git_sha": git_sha, - "battery": battery, - "surface": surface, - "driver": driver_name, - "provider": provider, - "classification": classification, - "model": model_id, - "requested_model": model_id, - "task_id": task["id"], - "author": task_author(task), - "rep": rep, - "success": False, - "verify_note": "", - "skipped": None, - "error": None, - "error_class": None, - "final_text": "", - "stop_reason": None, - "hit_max_iterations": False, - "result_pair_mismatch": False, - "token_count_failures": 0, - "result_tokens_estimated": None, - "calls": [], - "num_calls": 0, - "errored_calls": 0, - "alternate_calls": 0, - "out_of_set_calls": 0, - "total_result_tokens": 0, - "usage_per_iteration": [], - "cum_input_tokens": 0, - "wall_time_s": 0.0, - } - async def run_live( tasks: list[dict[str, Any]], @@ -553,434 +53,49 @@ async def run_live( resume: bool = False, record_result_payloads: bool = False, ) -> int: - surface = (surface or "full").strip().lower() - external = server_cmd is not None - if not external and surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " - "(or pass --server-cmd for an external surface)", - file=sys.stderr, - ) - return 2 - - driver_name = (driver_name or "api").strip().lower() - if driver_name not in KNOWN_DRIVERS: - print( - f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", - file=sys.stderr, - ) - return 2 - provider = (provider or "anthropic").strip().lower() - is_api_driver = driver_name in ("api", "sdk") - provider_id = provider if is_api_driver else None + """Delegate the legacy API while preserving its model-alias behavior.""" model_id = resolve_model_for_driver(driver_name, model_alias, provider=provider) - - run_id = uuid.uuid4().hex - git_sha = _git_sha() - battery = battery_fingerprint(tasks) - out_path.parent.mkdir(parents=True, exist_ok=True) - - resume_skip: set[tuple[str, int]] = set() - if resume: - try: - resume_skip, n_skip, n_retry = load_resume_skip_keys( - out_path, - surface=surface, - battery=battery, - model=model_id, - driver=driver_name, - provider=provider_id, - ) - except SystemExit as e: - print(e, file=sys.stderr) - return 2 - print(f"resume: skipping {n_skip} completed rows, retrying {n_retry}") - - # First line of a new/empty file is a meta header (skipped by loaders). - meta = make_run_meta_row( - run_id=run_id, + return await runner.run_live( + tasks, + model_alias=model_alias, + reps=reps, surface=surface, - battery=battery, - model=model_id, - driver=driver_name, - provider=provider_id, - git_sha=git_sha, + out_path=out_path, + driver_name=driver_name, + provider=provider, + server_cmd=server_cmd, + server_env=server_env, + resume=resume, + record_result_payloads=record_result_payloads, + resolved_model_id=model_id, ) - if maybe_write_run_meta(out_path, meta): - print(f"wrote meta header battery={battery} surface={surface}") - - plane, workspace_slug = make_plane_client() - # User chose --driver explicitly: codex live is allowed (they own the quota). - driver_kwargs: dict[str, Any] = {} - if is_api_driver: - driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) - if driver_name == "codex-cli": - driver_kwargs["allow_live"] = True - if not is_api_driver: - driver_kwargs["record_result_payloads"] = record_result_payloads - # --server-cmd must reach every driver; otherwise we - # silently benchmark the wrong server. - if server_cmd is not None: - driver_kwargs["server_command"] = server_cmd - driver = get_driver(driver_name, **driver_kwargs) - - print( - f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} model={model_id} " - f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" - ) - print(f"writing {out_path}") - - async def _run_tasks() -> None: - with out_path.open("a", encoding="utf-8") as fh: - for task in tasks: - if external: - # Foreign tool names have no overlay sets: no skips, no - # mispick classification — success/calls/errors only. - surface_sets = { - "skip": None, - "optimal_tools": set(), - "alternate_tools": set(), - "classification": "external", - } - else: - surface_sets = resolve_surface_tool_sets(task, surface) - for rep in range(reps): - if (task["id"], rep) in resume_skip: - print(f" {task['id']} rep={rep} RESUME_SKIP") - continue - - ctx: dict[str, Any] = {} - row = _base_row( - run_id=run_id, - git_sha=git_sha, - surface=surface, - driver_name=driver_name, - provider=provider_id, - model_id=model_id, - task=task, - rep=rep, - battery=battery, - classification=str(surface_sets["classification"]), - ) - try: - # Surface-unsupported tasks: record skip, no seed/agent. - if surface_sets.get("skip"): - reason = surface_sets["skip"] - row["skipped"] = reason - row["verify_note"] = reason - print(f" {task['id']} rep={rep} SKIPPED: {reason}") - else: - task_needs = set(task.get("needs") or set()) - # Seed wrap: TaskSkipped → skip; other failures → infra_seed. - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) - except TaskSkipped as skip: - row["skipped"] = skip.reason - row["verify_note"] = skip.reason - print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") - except Exception as exc: - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "infra_seed" - row["verify_note"] = "" - print( - f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - if ctx.get("project_name"): - print( - f" orphaned project may remain: {ctx['project_name']}", - file=sys.stderr, - ) - else: - if "bug_type" in task_needs and not ctx.get("bug_type"): - reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" - row["skipped"] = reason - row["verify_note"] = reason - print(f" {task['id']} rep={rep} SKIPPED: {reason}") - else: - agent: dict[str, Any] | None = None - # Agent wrap: API failures and CLI failures are infrastructure. - # Contained CLI stops (timeout / error subtypes) return AgentRun. - try: - agent = await run_agent_task_via_driver( - driver=driver, - model_id=model_id, - task=task, - ctx=ctx, - workspace_slug=workspace_slug, - surface=surface, - optimal_tools=surface_sets["optimal_tools"], - alternate_tools=surface_sets["alternate_tools"], - server_env=server_env, - ) - except PromptBindError as exc: - # Empty/missing seed IDs in the prompt — not an agent failure. - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "infra_seed" - row["verify_note"] = "" - print( - f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - agent = None - except Exception as exc: - if driver_name == "sdk": - agent_err_class = "infra_sdk" - elif is_api_driver: - agent_err_class = "infra_api" - else: - agent_err_class = "infra_cli" - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = agent_err_class - row["verify_note"] = "" - print( - f" {task['id']} rep={rep} ERROR[{agent_err_class}]: {exc}", - file=sys.stderr, - ) - agent = None - - if agent is not None: - row.update(agent) - if external: - # Empty overlay sets would classify every call - # out-of-set; null the counters instead. - row["alternate_calls"] = None - row["out_of_set_calls"] = None - - # CLI infra stops: timeout + error subtypes except error_max_turns. - stop_reason = agent.get("stop_reason") - if driver_name.endswith("-cli") and is_infra_cli_stop_reason( - str(stop_reason) if stop_reason is not None else None - ): - row["success"] = False - row["error_class"] = "infra_cli" - if stop_reason == "timeout": - row["error"] = _timeout_error_message(agent) - else: - notes = [ - n for n in (agent.get("driver_notes") or []) if isinstance(n, str) - ] - detail = "; ".join(notes) if notes else str(stop_reason) - row["error"] = detail - row["verify_note"] = "" - print( - f" {task['id']} rep={rep} ERROR[infra_cli]: {row['error']}", - file=sys.stderr, - ) - else: - verify = task["verify"] - try: - ok, note = await verify( - plane, - ctx, - { - "final_text": agent["final_text"], - "calls": agent["calls"], - }, - ) - row["success"] = bool(ok) - row["verify_note"] = note - print( - f" {task['id']} rep={rep} success={ok} " - f"calls={agent['num_calls']} note={note!r}" - ) - except TaskSkipped as skip: - row["skipped"] = skip.reason - row["verify_note"] = skip.reason - print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") - except Exception as exc: - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "task" - row["verify_note"] = "" - print( - f" {task['id']} rep={rep} ERROR[task]: {exc}", - file=sys.stderr, - ) - except Exception as exc: - # Anything outside seed/driver/verify wraps. - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "task" - row["verify_note"] = "" - print(f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr) - if ctx.get("project_name"): - print( - f" orphaned project may remain: {ctx['project_name']}", - file=sys.stderr, - ) - finally: - try: - teardown(plane, ctx) - except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) - if ctx.get("project_name"): - print(f" orphaned project: {ctx['project_name']}", file=sys.stderr) - fh.write(json.dumps(row, default=str) + "\n") - fh.flush() - await _run_tasks() - - return 0 - - -async def run_canary( - tasks: list[dict[str, Any]], - *, - surface: str, -) -> int: - """Seed + verify(empty agent) + teardown per task; no driver/model. - - Passes only when every verifier returns falsy ok on a do-nothing agent. - Any ok=True is a broken verifier (false positive). - """ - surface = (surface or "full").strip().lower() - if surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", - file=sys.stderr, - ) - return 2 - - battery = battery_fingerprint(tasks) - plane, _workspace_slug = make_plane_client() - print(f"canary battery={battery} surface={surface} tasks={[t['id'] for t in tasks]}") - - broken: list[str] = [] - verified_count = 0 - empty_agent = {"final_text": "", "calls": []} - - for task in tasks: - surface_sets = resolve_surface_tool_sets(task, surface) - if surface_sets.get("skip"): - print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") - continue - - ctx: dict[str, Any] = {} - task_needs = set(task.get("needs") or set()) - try: - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) - except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") - continue - if "bug_type" in task_needs and not ctx.get("bug_type"): - reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" - print(f" {task['id']} SKIPPED: {reason}") - continue - try: - ok, note = await task["verify"](plane, ctx, empty_agent) - except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") - continue - verified_count += 1 - if ok: - broken.append(task["id"]) - print(f" BROKEN VERIFIER: {task['id']} note={note!r}") - else: - print(f" {task['id']} ok=False note={note!r}") - except Exception as exc: - print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) - broken.append(task["id"]) - finally: - try: - teardown(plane, ctx) - except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) - - if broken: - for tid in broken: - print(f"BROKEN VERIFIER: {tid}", file=sys.stderr) - return 1 - if verified_count == 0: - print( - "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", - file=sys.stderr, - ) - return 1 - print(f"canary: all verifiers reject empty agent ({verified_count} verified)") - return 0 - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv) - - if args.list: - return cmd_list() - - ids = _task_ids(args.tasks) - try: - tasks = get_tasks(ids) - except SystemExit as e: - print(e, file=sys.stderr) - return 2 - - if args.dry_run: - return cmd_dry_run(tasks) - - surface = (args.surface or "full").strip().lower() - server_cmd: list[str] | None = None - if args.server_cmd: - import shlex - - server_cmd = shlex.split(args.server_cmd) - if not server_cmd: - print("error: --server-cmd is empty", file=sys.stderr) - return 2 - elif surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " - "(or pass --server-cmd for an external surface)", - file=sys.stderr, - ) - return 2 - - server_env: dict[str, str] = {} - for pair in args.server_env: - key, sep, val = pair.partition("=") - if not sep or not key: - print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) - return 2 - server_env[key] = val - - # Canary: live env only — no driver/model required. - if args.canary: - return asyncio.run(run_canary(tasks, surface=surface)) - - driver_name = (getattr(args, "driver", None) or "api").strip().lower() - if driver_name not in KNOWN_DRIVERS: - print( - f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", - file=sys.stderr, - ) - return 2 - - if args.resume: - out = Path(args.resume) - elif args.out: - out = Path(args.out) - else: - out = DEFAULT_OUT_DIR / f"{uuid.uuid4().hex}.jsonl" - - return asyncio.run( - run_live( - tasks, - model_alias=args.model, - reps=args.reps, - surface=surface, - out_path=out, - driver_name=driver_name, - provider=args.provider, - server_cmd=server_cmd, - server_env=server_env or None, - resume=bool(args.resume), - record_result_payloads=bool(args.record_result_payloads), - ) - ) +__all__ = [ + "API_MODEL_ALIASES", + "CLI_MODEL_ALIASES", + "DEFAULT_OUT_DIR", + "KNOWN_SURFACES", + "MAX_ITERATIONS", + "MAX_TOKENS", + "MODEL_ALIASES", + "classify_call", + "cmd_dry_run", + "cmd_list", + "is_infra_cli_stop_reason", + "is_meta_or_non_task_row", + "load_resume_skip_keys", + "main", + "make_run_meta_row", + "maybe_write_run_meta", + "parse_args", + "resolve_model_for_driver", + "run_agent_task_via_driver", + "run_canary", + "run_live", + "should_skip_resume_row", + "stdio_server_env", +] if __name__ == "__main__": diff --git a/evals/runner.py b/evals/runner.py new file mode 100644 index 00000000..5b147c1e --- /dev/null +++ b/evals/runner.py @@ -0,0 +1,746 @@ +"""Live execution, resume bookkeeping, row assembly, and verifier canary.""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.drivers import ( + KNOWN_DRIVERS, + agent_run_to_harness_dict, + get_driver, +) +from evals.seed import make_plane_client, seed, teardown +from evals.tasks import ( + PromptBindError, + TaskSkipped, + battery_fingerprint, + format_task_prompt, + resolve_surface_tool_sets, + task_author, +) + +# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). +# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env +# (see plane_mcp.v2.choose_stdio_mcp). +KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) + +MAX_ITERATIONS = 15 +MAX_TOKENS = 8192 + + +def _git_sha() -> str: + try: + return ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + cwd=Path(__file__).resolve().parent.parent, + ) + .decode() + .strip() + ) + except Exception: + return "unknown" + + +def _system_preamble(workspace_slug: str, project_name: str) -> str: + """Keep under 100 words — part of measured context.""" + return ( + f"You are evaluating Plane project management tools. " + f"Workspace slug: {workspace_slug}. Project name: {project_name}. " + f"Complete the task using the available tools, then stop." + ) + + +def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: + if tool in optimal: + return "optimal" + if tool in alternate: + return "alternate" + return "out_of_set" + + +def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6). + + ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the + v2 tool registry. ``surface=full`` leaves the var unset (legacy default). + Other surface names (external servers under benchmark) set nothing; their + selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. + """ + env: dict[str, str] = {} + if path := os.environ.get("PATH"): + env["PATH"] = path + if home := os.environ.get("HOME"): + env["HOME"] = home + env["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] + env["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + env["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if surface == "v2": + env["PLANE_MCP_SURFACE"] = "v2" + elif surface == "v2-schema": + env["PLANE_MCP_SURFACE"] = "v2-schema" + if extra: + env.update(extra) + return env + + +def should_skip_resume_row(row: dict[str, Any]) -> bool: + """Return True if a prior row is a completed result that resume should skip. + + Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. + Rows with ``skipped`` set are treated as complete and are not retried (intentional: + surface/plan skips are stable outcomes, not infra failures). + Pure function — unit-tested without the live battery. + """ + ec = row.get("error_class") + if isinstance(ec, str) and ec.startswith("infra_"): + return False + if row.get("error") is not None: + return False + return True + + +def _resume_field_mismatch( + row: dict[str, Any], + *, + field: str, + expected: str | None, +) -> str | None: + """Return an error message if row[field] is present and disagrees with expected.""" + if expected is None: + return None + raw = row.get(field) + if raw is None or raw == "": + return None # back-compat: older rows without the key pass + # Surface/driver/provider compare case-insensitively; battery/model are exact. + if field in ("surface", "driver", "provider"): + got, want = str(raw).strip().lower(), expected.strip().lower() + else: + got, want = str(raw).strip(), expected.strip() + if got != want: + return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" + return None + + +def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: + """True when a CLI AgentRun stop_reason should be classified as infra_cli. + + ``timeout`` and Claude error subtypes (``error_during_execution``, bare + ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task + failure and stays in the success-rate denominator. + """ + if not stop_reason: + return False + sr = str(stop_reason) + if sr == "timeout": + return True + if sr == "error_max_turns": + return False + if sr == "error" or sr.startswith("error_"): + return True + return False + + +def _timeout_error_message(agent: dict[str, Any]) -> str: + """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" + for note in agent.get("driver_notes") or []: + if isinstance(note, str) and note.startswith("timeout after"): + return note + return "timeout" + + +def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: + """True for run-header meta lines or any row without a task_id.""" + if row.get("row_type") == "meta": + return True + return row.get("task_id") is None + + +def load_resume_skip_keys( + path: Path, + *, + surface: str, + battery: str | None = None, + model: str | None = None, + driver: str | None = None, + provider: str | None = None, +) -> tuple[set[tuple[str, int]], int, int]: + """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. + + Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` + (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / + battery / model / driver / provider disagrees with the current run (missing keys pass for + back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked + but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. + """ + if not path.is_file(): + return set(), 0, 0 + skip_keys: set[tuple[str, int]] = set() + seen: set[tuple[str, int]] = set() + with path.open(encoding="utf-8") as fh: + for line_no, line in enumerate(fh, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: --resume {path}:{line_no}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict): + continue + for field, expected in ( + ("surface", surface), + ("battery", battery), + ("driver", driver), + ("provider", provider), + ): + msg = _resume_field_mismatch(row, field=field, expected=expected) + if msg: + raise SystemExit(msg) + # New API rows keep both the requested ID (resume identity) and the + # provider-reported model that actually ran. Older rows only have model. + model_row = dict(row) + if model_row.get("requested_model"): + model_row["model"] = model_row["requested_model"] + msg = _resume_field_mismatch(model_row, field="model", expected=model) + if msg: + raise SystemExit(msg) + # Meta / header rows: checked above, not part of resume key set. + if is_meta_or_non_task_row(row): + continue + tid = row.get("task_id") + rep = row.get("rep") + if tid is None or rep is None: + continue + key = (str(tid), int(rep)) + seen.add(key) + if should_skip_resume_row(row): + skip_keys.add(key) + else: + # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. + skip_keys.discard(key) + n_retry = len(seen - skip_keys) + return skip_keys, len(skip_keys), n_retry + + +def make_run_meta_row( + *, + run_id: str, + surface: str, + battery: str, + model: str | None, + driver: str, + git_sha: str, + provider: str | None = None, + ts: str | None = None, +) -> dict[str, Any]: + """Build the single first-line meta record for a new output JSONL.""" + return { + "row_type": "meta", + "run_id": run_id, + "surface": surface, + "battery": battery, + "model": model, + "driver": driver, + "provider": provider, + "git_sha": git_sha, + "ts": ts or datetime.now(timezone.utc).isoformat(), + } + + +def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: + """Write meta as the first line when the file is missing or empty. Returns True if written.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file() and path.stat().st_size > 0: + return False + with path.open("w", encoding="utf-8") as fh: + fh.write(json.dumps(meta, default=str) + "\n") + return True + + +async def run_agent_task_via_driver( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + ctx: dict[str, Any], + workspace_slug: str, + surface: str = "full", + optimal_tools: set[str] | None = None, + alternate_tools: set[str] | None = None, + server_env: dict[str, str] | None = None, +) -> dict[str, Any]: + """Run one task through the selected AgentDriver.""" + project_name = ctx["project_name"] + system = _system_preamble(workspace_slug, project_name) + prompt = format_task_prompt(task, ctx, strict=True) + optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) + alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) + assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" + + mcp_env = stdio_server_env(surface=surface, extra=server_env) + # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. + agent_run = await asyncio.to_thread( + driver.run_task, + prompt, + mcp_env, + model_id, + MAX_ITERATIONS, + system=system, + cwd=Path(__file__).resolve().parent.parent, + ) + return agent_run_to_harness_dict( + agent_run, + optimal=optimal, + alternate=alternate, + classify=classify_call, + ) + + +def _base_row( + *, + run_id: str, + git_sha: str, + surface: str, + driver_name: str, + provider: str | None, + model_id: str | None, + task: dict[str, Any], + rep: int, + battery: str, + classification: str, +) -> dict[str, Any]: + return { + "run_id": run_id, + "ts": datetime.now(timezone.utc).isoformat(), + "git_sha": git_sha, + "battery": battery, + "surface": surface, + "driver": driver_name, + "provider": provider, + "classification": classification, + "model": model_id, + "requested_model": model_id, + "task_id": task["id"], + "author": task_author(task), + "rep": rep, + "success": False, + "verify_note": "", + "skipped": None, + "error": None, + "error_class": None, + "final_text": "", + "stop_reason": None, + "hit_max_iterations": False, + "result_pair_mismatch": False, + "token_count_failures": 0, + "result_tokens_estimated": None, + "calls": [], + "num_calls": 0, + "errored_calls": 0, + "alternate_calls": 0, + "out_of_set_calls": 0, + "total_result_tokens": 0, + "usage_per_iteration": [], + "cum_input_tokens": 0, + "wall_time_s": 0.0, + } + + +async def run_live( + tasks: list[dict[str, Any]], + *, + model_alias: str, + reps: int, + surface: str, + out_path: Path, + driver_name: str = "api", + provider: str = "anthropic", + server_cmd: list[str] | None = None, + server_env: dict[str, str] | None = None, + resume: bool = False, + record_result_payloads: bool = False, + resolved_model_id: str | None = None, +) -> int: + surface = (surface or "full").strip().lower() + external = server_cmd is not None + if not external and surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " + "(or pass --server-cmd for an external surface)", + file=sys.stderr, + ) + return 2 + + driver_name = (driver_name or "api").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + provider = (provider or "anthropic").strip().lower() + is_api_driver = driver_name in ("api", "sdk") + provider_id = provider if is_api_driver else None + model_id = resolved_model_id if resolved_model_id is not None else model_alias + + run_id = uuid.uuid4().hex + git_sha = _git_sha() + battery = battery_fingerprint(tasks) + out_path.parent.mkdir(parents=True, exist_ok=True) + + resume_skip: set[tuple[str, int]] = set() + if resume: + try: + resume_skip, n_skip, n_retry = load_resume_skip_keys( + out_path, + surface=surface, + battery=battery, + model=model_id, + driver=driver_name, + provider=provider_id, + ) + except SystemExit as e: + print(e, file=sys.stderr) + return 2 + print(f"resume: skipping {n_skip} completed rows, retrying {n_retry}") + + # First line of a new/empty file is a meta header (skipped by loaders). + meta = make_run_meta_row( + run_id=run_id, + surface=surface, + battery=battery, + model=model_id, + driver=driver_name, + provider=provider_id, + git_sha=git_sha, + ) + if maybe_write_run_meta(out_path, meta): + print(f"wrote meta header battery={battery} surface={surface}") + + plane, workspace_slug = make_plane_client() + # User chose --driver explicitly: codex live is allowed (they own the quota). + driver_kwargs: dict[str, Any] = {} + if is_api_driver: + driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) + if driver_name == "codex-cli": + driver_kwargs["allow_live"] = True + if not is_api_driver: + driver_kwargs["record_result_payloads"] = record_result_payloads + # --server-cmd must reach every driver; otherwise we + # silently benchmark the wrong server. + if server_cmd is not None: + driver_kwargs["server_command"] = server_cmd + driver = get_driver(driver_name, **driver_kwargs) + + print( + f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} model={model_id} " + f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" + ) + print(f"writing {out_path}") + + async def _run_tasks() -> None: + with out_path.open("a", encoding="utf-8") as fh: + for task in tasks: + if external: + # Foreign tool names have no overlay sets: no skips, no + # mispick classification — success/calls/errors only. + surface_sets = { + "skip": None, + "optimal_tools": set(), + "alternate_tools": set(), + "classification": "external", + } + else: + surface_sets = resolve_surface_tool_sets(task, surface) + for rep in range(reps): + if (task["id"], rep) in resume_skip: + print(f" {task['id']} rep={rep} RESUME_SKIP") + continue + + ctx: dict[str, Any] = {} + row = _base_row( + run_id=run_id, + git_sha=git_sha, + surface=surface, + driver_name=driver_name, + provider=provider_id, + model_id=model_id, + task=task, + rep=rep, + battery=battery, + classification=str(surface_sets["classification"]), + ) + try: + # Surface-unsupported tasks: record skip, no seed/agent. + if surface_sets.get("skip"): + reason = surface_sets["skip"] + row["skipped"] = reason + row["verify_note"] = reason + print(f" {task['id']} rep={rep} SKIPPED: {reason}") + else: + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) + except TaskSkipped as skip: + row["skipped"] = skip.reason + row["verify_note"] = skip.reason + print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") + except Exception as exc: + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "infra_seed" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + if ctx.get("project_name"): + print( + f" orphaned project may remain: {ctx['project_name']}", + file=sys.stderr, + ) + else: + if "bug_type" in task_needs and not ctx.get("bug_type"): + reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" + row["skipped"] = reason + row["verify_note"] = reason + print(f" {task['id']} rep={rep} SKIPPED: {reason}") + else: + agent: dict[str, Any] | None = None + # Agent wrap: API failures and CLI failures are infrastructure. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + agent = await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=ctx, + workspace_slug=workspace_slug, + surface=surface, + optimal_tools=surface_sets["optimal_tools"], + alternate_tools=surface_sets["alternate_tools"], + server_env=server_env, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "infra_seed" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + agent = None + except Exception as exc: + if driver_name == "sdk": + agent_err_class = "infra_sdk" + elif is_api_driver: + agent_err_class = "infra_api" + else: + agent_err_class = "infra_cli" + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = agent_err_class + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[{agent_err_class}]: {exc}", + file=sys.stderr, + ) + agent = None + + if agent is not None: + row.update(agent) + if external: + # Empty overlay sets would classify every call + # out-of-set; null the counters instead. + row["alternate_calls"] = None + row["out_of_set_calls"] = None + + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.get("stop_reason") + if driver_name.endswith("-cli") and is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): + row["success"] = False + row["error_class"] = "infra_cli" + if stop_reason == "timeout": + row["error"] = _timeout_error_message(agent) + else: + notes = [ + n for n in (agent.get("driver_notes") or []) if isinstance(n, str) + ] + detail = "; ".join(notes) if notes else str(stop_reason) + row["error"] = detail + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[infra_cli]: {row['error']}", + file=sys.stderr, + ) + else: + verify = task["verify"] + try: + ok, note = await verify( + plane, + ctx, + { + "final_text": agent["final_text"], + "calls": agent["calls"], + }, + ) + row["success"] = bool(ok) + row["verify_note"] = note + print( + f" {task['id']} rep={rep} success={ok} " + f"calls={agent['num_calls']} note={note!r}" + ) + except TaskSkipped as skip: + row["skipped"] = skip.reason + row["verify_note"] = skip.reason + print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") + except Exception as exc: + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "task" + row["verify_note"] = "" + print( + f" {task['id']} rep={rep} ERROR[task]: {exc}", + file=sys.stderr, + ) + except Exception as exc: + # Anything outside seed/driver/verify wraps. + row["success"] = False + row["error"] = f"{type(exc).__name__}: {exc}" + row["error_class"] = "task" + row["verify_note"] = "" + print(f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr) + if ctx.get("project_name"): + print( + f" orphaned project may remain: {ctx['project_name']}", + file=sys.stderr, + ) + finally: + try: + teardown(plane, ctx) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + if ctx.get("project_name"): + print(f" orphaned project: {ctx['project_name']}", file=sys.stderr) + + fh.write(json.dumps(row, default=str) + "\n") + fh.flush() + + await _run_tasks() + + return 0 + + +async def run_canary( + tasks: list[dict[str, Any]], + *, + surface: str, +) -> int: + """Seed + verify(empty agent) + teardown per task; no driver/model. + + Passes only when every verifier returns falsy ok on a do-nothing agent. + Any ok=True is a broken verifier (false positive). + """ + surface = (surface or "full").strip().lower() + if surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", + file=sys.stderr, + ) + return 2 + + battery = battery_fingerprint(tasks) + plane, _workspace_slug = make_plane_client() + print(f"canary battery={battery} surface={surface} tasks={[t['id'] for t in tasks]}") + + broken: list[str] = [] + verified_count = 0 + empty_agent = {"final_text": "", "calls": []} + + for task in tasks: + surface_sets = resolve_surface_tool_sets(task, surface) + if surface_sets.get("skip"): + print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") + continue + + ctx: dict[str, Any] = {} + task_needs = set(task.get("needs") or set()) + try: + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + if "bug_type" in task_needs and not ctx.get("bug_type"): + reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" + print(f" {task['id']} SKIPPED: {reason}") + continue + try: + ok, note = await task["verify"](plane, ctx, empty_agent) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + verified_count += 1 + if ok: + broken.append(task["id"]) + print(f" BROKEN VERIFIER: {task['id']} note={note!r}") + else: + print(f" {task['id']} ok=False note={note!r}") + except Exception as exc: + print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) + broken.append(task["id"]) + finally: + try: + teardown(plane, ctx) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + + if broken: + for tid in broken: + print(f"BROKEN VERIFIER: {tid}", file=sys.stderr) + return 1 + if verified_count == 0: + print( + "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", + file=sys.stderr, + ) + return 1 + print(f"canary: all verifiers reject empty agent ({verified_count} verified)") + return 0 + + +__all__ = [ + "KNOWN_SURFACES", + "MAX_ITERATIONS", + "MAX_TOKENS", + "classify_call", + "is_infra_cli_stop_reason", + "is_meta_or_non_task_row", + "load_resume_skip_keys", + "make_run_meta_row", + "maybe_write_run_meta", + "run_agent_task_via_driver", + "run_canary", + "run_live", + "should_skip_resume_row", + "stdio_server_env", +] diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 4e5f07ea..90976470 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -14,6 +14,7 @@ from evals import report as report_mod from evals import run as run_mod +from evals import runner as runner_mod from evals import seed as seed_mod from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize @@ -182,14 +183,14 @@ def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) def boom_seed(plane, run_id, needs, ctx): ctx["project_name"] = "EVAL deadbeef" raise HttpError("identifier already taken", 409) - monkeypatch.setattr(run_mod, "seed", boom_seed) - monkeypatch.setattr(run_mod, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_mod, "seed", boom_seed) + monkeypatch.setattr(runner_mod, "teardown", lambda plane, ctx: None) task = { "id": "T1", @@ -226,13 +227,13 @@ def boom_seed(plane, run_id, needs, ctx): def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) def ok_seed(plane, run_id, needs, ctx): ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) - monkeypatch.setattr(run_mod, "seed", ok_seed) - monkeypatch.setattr(run_mod, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_mod, "seed", ok_seed) + monkeypatch.setattr(runner_mod, "teardown", lambda plane, ctx: None) class BoomDriver: name = "claude-cli" @@ -240,7 +241,7 @@ class BoomDriver: def run_task(self, *args, **kwargs): raise RuntimeError("claude cli failed: json_parse_failed") - monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: BoomDriver()) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: BoomDriver()) task = { "id": "T2", @@ -275,9 +276,9 @@ def test_run_live_timeout_agent_is_infra_cli(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) class TimeoutDriver: name = "claude-cli" @@ -291,7 +292,7 @@ def run_task(self, *args, **kwargs): notes=["timeout after 900s"], ) - monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: TimeoutDriver()) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: TimeoutDriver()) verify_calls: list[Any] = [] @@ -339,9 +340,9 @@ def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatc """exit 1 + parseable JSON subtype error_during_execution → infra_cli; verify not called.""" out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) payload = { "type": "result", @@ -356,7 +357,7 @@ def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatc def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") - monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) verify_calls: list[Any] = [] @@ -396,9 +397,9 @@ def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): """exit 1 + subtype error_max_turns stays in the task denominator (not infra_cli).""" out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) payload = { "type": "result", @@ -413,7 +414,7 @@ def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") - monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) verify_calls: list[Any] = [] @@ -777,9 +778,9 @@ def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): def test_canary_detects_broken_verifier(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) async def always_ok(plane, ctx, run): return True, "false positive" @@ -813,9 +814,9 @@ async def correctly_fails(plane, ctx, run): def test_canary_passes_when_all_verifiers_reject(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) async def reject(plane, ctx, run): assert run == {"final_text": "", "calls": []} @@ -838,11 +839,11 @@ async def reject(plane, ctx, run): def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: None) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) monkeypatch.setattr( - run_mod, + runner_mod, "resolve_surface_tool_sets", lambda task, surface: { "skip": "unsupported on surface", @@ -901,7 +902,7 @@ def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypat out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") fake_plane = MagicMock() - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) seed_calls: list[str] = [] def ok_seed(plane, run_id, needs, ctx): @@ -909,8 +910,8 @@ def ok_seed(plane, run_id, needs, ctx): ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) seed_calls.append(run_id) - monkeypatch.setattr(run_mod, "seed", ok_seed) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "seed", ok_seed) + monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) class OkDriver: name = "claude-cli" @@ -923,7 +924,7 @@ def run_task(self, *args, **kwargs): stopped_reason="end_turn", ) - monkeypatch.setattr(run_mod, "get_driver", lambda name, **kw: OkDriver()) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: OkDriver()) async def verify_ok(plane, ctx, run): return True, "ok" diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index c720c0b0..aaf89426 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -1684,7 +1684,7 @@ def fake_run(cmd, **kwargs): def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): """--server-cmd must not be Claude-only.""" - from evals import run as run_mod + from evals import runner as run_mod captured: dict = {} diff --git a/tests/test_evals_surface.py b/tests/test_evals_surface.py index 32f8d907..b8a7a945 100644 --- a/tests/test_evals_surface.py +++ b/tests/test_evals_surface.py @@ -114,7 +114,7 @@ def test_classify_uses_resolved_sets(): def test_skip_path_no_network(monkeypatch): """Unsupported surface skip must not call seed/teardown/agent.""" - from evals import run as run_mod + from evals import runner as run_mod seeded = [] torn = [] From adf653458ed5788e58acc5e2e9751143df942a5d Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 02:15:24 +0530 Subject: [PATCH 07/93] Reconcile the eval docs with the code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four phases of refactor left DESIGN.md instructing readers to build the driver on the Anthropic beta tool_runner and "not improvise alternatives" — which is exactly what we did, and for reasons the document should now explain. It also still described the harness as a walking skeleton with a section on what was not built yet. It is now the rationale for what exists: what each metric buys, why verification reads the API back instead of grading prose, why call counts come off the wire, and where the driver/backend seam sits. Roadmap sections are deleted rather than updated; git history is the roadmap. The README's prerequisites claimed a Business/Enterprise licence was required. It is not: the mock flag server enables every flag regardless, and a workspace with no licence row seeds every fixture — checked by canary rather than argued. Restored with sharper wording: the feature-flag cache trap (a cached answer from the wrong flag server makes gated endpoints 402 and looks exactly like a plan problem, so it is worth checking first), and the fact that Codex rejects the short model aliases the harness forwards unchanged. Co-Authored-By: Claude Fable 5 --- evals/DESIGN.md | 518 ++++++++++++++++++++---------------------------- evals/README.md | 132 +++++++----- 2 files changed, 293 insertions(+), 357 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 7c8e3b24..d8c45f97 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -1,321 +1,229 @@ # Plane MCP Tool-Surface Eval Harness -Measures how well an LLM agent completes real Plane tasks through this MCP server's tool -surface. Produces decision-grade numbers for the tool-consolidation discussion: success rate, -tool calls to done, wrong-tool picks, and per-call response token cost. - -This document is the full spec. Phase 1 (walking skeleton) implements a subset — see -"Phase 1 scope" at the bottom. - -## Why - -The live surface is 177 tools. A consolidation proposal (139→47) exists, but its cost -claims were estimate-based and wrong by 3.4× when measured. Before reshaping anything we -need empirical answers to: - -1. **Mispick rate** — how often does an agent choose the wrong tool among overlapping ones - (7 list-variants for work items, links vs relations, etc.)? -2. **Calls-to-done vs optimal** — how much does the name→UUID resolution dance and - sub-object fan-out (item + comments + links as separate calls) cost? -3. **Response bloat** — how many tokens does each tool result actually inject into context? - -The harness must support A/B comparison: same tasks against different tool surfaces -(`full` today; later `core` tag-filter and `v2` transform-layer variants). - -## Architecture - -**Driver:** `ApiDriver` owns the model/tool loop and the stdio MCP session. Provider adapters -own only conversation state and wire translation behind a neutral `ModelBackend` protocol: -`start(system, prompt, tools)`, `next_turn()`, and `add_tool_results(results)`. Anthropic uses -the stable Messages API; OpenAI uses Chat Completions function tools. CLI drivers keep the -same `AgentDriver` boundary, so `run.py` has one execution path for every driver. - -Neutral turns contain final text, tool calls keyed by provider call ID, normalized usage, -and stop reason. The driver records calls, executes MCP tools, pairs results back by call ID, -and passes neutral results to the backend for the next provider turn. This isolates tool- -surface behavior without a coding-harness system prompt or built-in tools. - -Notes: -- One fresh stdio server subprocess per task run (cheap, isolates state). -- Record provider usage from **every** model turn (input tokens, output tokens, - cache_read_input_tokens, cache_creation_input_tokens) — this is the exact context cost, - returned free; the per-result counts below are a size proxy, not the cost figure. -- Record the final message's `stop_reason`, and whether the loop ended by exhausting - `max_iterations` — a capped/truncated run must be distinguishable from a genuine failure. - Detect the cap from `stop_reason` (a model can legitimately finish with `end_turn` on - exactly its last permitted iteration — an unconditional iteration-count check misreports - that as capped). -- **Never execute tools on a refusal-terminated turn** (`stop_reason == "refusal"`), even if - the response also contains tool calls. Record the calls for auditability, then stop. -- Pair every tool result to its call by call ID, never list position. Missing, duplicate, or - unknown IDs set `result_pair_mismatch`. -- `wall_time_s` measures the agent loop only: start the clock after `list_tools()` returns, - stop it when the loop exits — MCP subprocess spawn/teardown and post-loop token counting - are excluded. -- Build the stdio server env **from scratch** (`PATH`, `HOME`, plus exactly the three - `PLANE_*` vars) — never inherit `os.environ`. `plane_mcp/client.py` prefers - `PLANE_INTERNAL_BASE_URL` over `PLANE_BASE_URL`, so an inherited value silently points - the agent at a different Plane instance than seed/verify. -- A harness/API failure (provider exception, MCP crash) is recorded as `error: ""` on the - row — it is neither a task failure nor a skip, and the row's zeroed metrics must not - enter any statistic. -- `SYSTEM_PREAMBLE` names the eval workspace slug and project name, states "complete the - task using the available tools, then stop", and nothing else. Keep it under 100 words — - it is part of the measured context. -- Omit thinking and sampling parameters; the API backend only sets the model, token cap, - system/instructions, conversation, and tools. -- Final assistant text = the last model turn's text (used by read-task verifiers). - -**Token sizing of tool results:** the owned loop holds the complete text passed back to the -model, so `result_chars` is always exact. A backend may expose a token counter; otherwise the -driver uses a deterministic character estimate and sets `result_tokens_estimated: true` on -the row. No provider token-count endpoint is required per tool result. - -CLI rows use the same shared character estimator over proxy-recorded `result_chars`. Optional -payload recording permits local tokenizer counting in the parent harness, but stays off by -default; the stdlib-only proxy does not import tokenizers. - -Rules: -- Run these counts **after** the agent loop finishes, not inline — they must not pollute - `wall_time_s`. Buffer the raw result strings during the run, count at the end. -- A tool result's content may be a list of blocks. Text-only blocks are concatenated; - non-text or mixed content is serialized into the exact string sent to the model and marked - `result_kind: "image"` or `"mixed"`. Error results are sized like successful results. -- Also record raw `len(chars)` alongside every count. - -**API model aliases** (provider-specific): - -| provider | alias | model id | role | -|---|---|---|---| -| Anthropic | `sonnet` | `claude-sonnet-5` | default / representative agent | -| Anthropic | `haiku` | `claude-haiku-4-5` | weaker-model canary | -| OpenAI | `sonnet` | `gpt-5` | representative agent | -| OpenAI | `haiku` | `gpt-5-mini` | faster/weaker-model canary | - -## Environment - -| var | purpose | -|---|---| -| `ANTHROPIC_API_KEY` | default API-provider authentication | -| `OPENAI_API_KEY` | OpenAI provider authentication when its optional SDK is installed | -| `EVAL_PLANE_API_KEY` | Plane API key for the **dedicated eval workspace** | -| `EVAL_PLANE_WORKSPACE_SLUG` | eval workspace slug — never a production workspace | -| `EVAL_PLANE_BASE_URL` | optional, defaults to `https://api.plane.so` | - -Seeding and verification talk to Plane directly via `plane-sdk` (already a dependency). -Construct the client the same way `plane_mcp/client.py` does for stdio mode, but from the -`EVAL_*` vars. - -## Files +This harness measures how well an LLM agent completes real Plane tasks through an MCP +tool surface. It exists to replace predictions about a surface with observations from +actual agent runs: whether the task succeeded, how many Plane calls it took, which tools +were selected, and how much tool-result content was returned to the model. -``` -evals/ - __init__.py - DESIGN.md # this file - drivers/api/ # owned loop + neutral, Anthropic, and OpenAI backends - tasks.py # task definitions (plain dicts) + verifier functions - seed.py # per-run fixture create/teardown via plane-sdk - run.py # CLI driver (python -m evals.run) - report.py # summary table + A/B delta (python -m evals.report) - results/ # *.jsonl output — gitignored -``` +This document explains why the harness is shaped this way. Operational commands live in +`evals/README.md`. -Dependencies: add to `pyproject.toml`: +## The questions it answers -```toml -[project.optional-dependencies] -evals = ["anthropic>=0.121.0"] -``` +The original tool-consolidation question breaks down into three measurable questions: -Pin a `>=` floor for the stable Anthropic Messages client. OpenAI support stays optional: -the module imports its SDK lazily only when that provider is selected, and tests inject a -fake client. Do NOT change the existing `mcp==1.26.0` pin. - -## Task schema (`tasks.py`) - -Plain dicts, no classes: - -```python -{ - "id": "W1", - "tags": {"write", "tier1"}, - "prompt": "Create a work item in project {project}: title 'Login page 500s on empty " - "password', priority urgent, assign it to me, and add the 'auth' label.", - "optimal_calls": 4, - "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, - "alternate_tools": { - "search_work_items", - "list_states", - "retrieve_project", - "get_workspace_members", - "manage_work_item_assignee", - "manage_work_item_label", - "update_work_item", - }, - "needs": {"labels"}, # fixture groups seed.py must create - "verify": verify_w1, # async (plane, ctx, run) -> (bool, note) -} -``` +1. **Mispick rate** — how often does an agent choose an alternate or out-of-set tool when + several tools have overlapping names or capabilities? +2. **Calls-to-done versus optimal** — how much lookup, name-to-ID resolution, and + sub-object fan-out does the surface require before the task is complete? +3. **Response bloat** — how much tool-result content is injected into the conversation? -- `{project}` in prompts is formatted with the seeded project name at runtime. -- `optimal_tools` and `alternate_tools` are **disjoint** sets. Every call is classified as - one of `optimal` / `alternate` / `out_of_set`, plus an independent `is_error` flag. - **Mispick = alternate + out_of_set** — this is the eval's headline metric, so authoring - matters: a tool that works but is the *wrong pick among overlapping variants* (a list - variant where search is optimal, a link where a relation is asked for) belongs in - `alternate_tools` or nowhere, never in `optimal_tools`. The full ordered call list is - kept in the JSONL so classifications can be re-derived offline if sets are revised. -- **Action-dispatch surfaces need a finer mispick unit** (added 2026-08-11, for the P2 - A/B that compares PR #195's 29-tool `action`-multiplexer variant): on such a surface - the model almost always picks the "right" *tool* and fails inside it — wrong `action`, - or params invalid for the chosen action. Tool-name classification alone would - under-count exactly that failure mode and bias the A/B toward consolidation. Rule: when - a surface under test multiplexes verbs through a parameter, the classification unit is - `(tool, action)` — task authors list optimal/alternate *(tool, action)* pairs — and a - schema-valid call whose params are invalid for its declared action counts as - `out_of_set`, not merely `is_error`. Flat surfaces are unaffected (their action unit is - the tool name). The stored ordered call list already carries arguments, so this scoring - can also be re-derived retroactively. -- `verify` receives `run = {"final_text": str, "calls": [...]}` alongside the plane client - and seed ctx. Write tasks assert end state through the Plane API; read tasks match the - final text against seeded facts using **word-boundary regexes on exact seeded values** - (each verifier states its matching rule in a comment — naive substring matching is a - known false-positive source, e.g. bare "4" inside "24"). Resolve expected values via - API at verify time — never hardcode sequence numbers or UUIDs. - -## Fixtures (`seed.py`) - -Per run: create project named `EVAL {run8}` (`run8` = first 8 hex chars of `uuid4().hex`; -identifier `EV` + 4 of those hex chars uppercased, ≤12 chars) in the eval workspace. -Unique-per-run naming makes runs parallel-safe and crash-visible. Provide -`seed(plane, run_id, needs) -> ctx` and `teardown(plane, ctx)`; `ctx` carries project_id, -project name, and IDs of everything seeded. - -`seed()` must guarantee teardown information even on partial failure: it mutates a -caller-provided ctx in place (or raises with the partial ctx attached), so a failure -after project creation never leaks an untracked project. Feature probes must read the -keys the API actually returns (workspace toggle: `is_work_item_types_enabled`) and a -seed failure must fail loudly — never masquerade as a plan-gate skip. - -The R1 target item is seeded into a **non-default state** (e.g. a `started`-group state) -so a guessed default state name cannot pass verification. - -Teardown deletes the project **plus every workspace-scoped object seeded** — customers -(and any other object that survives project deletion) are tracked in `ctx` and deleted by -ID explicitly; project deletion alone is not sufficient cleanup. - -Seed only the fixture groups the selected tasks declare in `needs`: - -| group | contents | -|---|---| -| `items` | ~12 work items with fixed titles/priorities incl. "Payment webhook drops retries" (urgent); exactly 4 urgent open items total | -| `labels` | labels `auth`, `triage`, `perf` | -| `cycles` | "Sprint 12" (past-dated), "Sprint 13" (current) | -| `module` | "Checkout revamp" with 3 completed items | -| `bug_type` | work item type "Bug" — **plan-gated feature**: if the API rejects creation, seed() records `bug_type: None` and dependent tasks are SKIPPED (recorded in JSONL with `"skipped": reason`), not failed | -| `intake` | 2 intake items (one billing request, one obvious spam) | -| `customer` | customer "Acme Corp" + request "SSO support" | -| `release` | release "1.2.0" with 2 changelog entries | - -## Runner CLI (`run.py`) +Success is the guardrail around all three. A surface that uses fewer calls or returns less +text but fails the task is not an improvement. Conversely, success rate alone hides +avoidable calls, wrong turns, and large responses. The harness therefore records all four +dimensions for the same task execution. -``` -python -m evals.run --tasks R1,W1,S1 --model sonnet --reps 1 \ - --surface full --out evals/results/.jsonl -python -m evals.run --list # print task table, no network -python -m evals.run --dry-run --tasks R1 # print resolved prompt + seed plan, no network +The point is empirical comparison. Given the same task battery, model, and repetitions, +different surfaces can be compared from observed behavior rather than from tool counts, +schema inspection, or projected costs. The battery fingerprint records the prompt and +tool-set definition used for a run so incompatible batteries are not silently compared. + +## What is measured + +### Success + +Each task has an asynchronous verifier. Mutation tasks read Plane back through the API and +check the resulting state. Read tasks compare the final assistant text with facts obtained +from the seeded context or resolved through the API, using explicit answer contracts and +exact-value matchers where the task defines them. + +This avoids using the agent's explanation, confidence, or self-reported completion as the +source of truth. The model is also not asked to grade another model. Verification is tied to +the fixture and the Plane state the task was meant to affect. The canary runs every eligible +verifier against an empty agent result and fails if a do-nothing run passes. + +Skipped tasks and infrastructure failures are recorded separately. The report excludes +both from success denominators; a plan gate, unavailable fixture, provider failure, or MCP +process failure is not rewritten as an agent task failure. + +### Calls to done + +`num_calls` counts Plane MCP calls made during the task. Each catalog entry also declares an +`optimal_calls` baseline. The report shows the observed distribution rather than assuming +one run is representative. + +Client-local tools such as shell or tool-search helpers are retained separately as +`client_tool_calls`; they do not count as Plane calls. For an external server launched with +`--server-cmd`, call counts still apply, but the runner marks classification as `external` +and clears the row-level alternate/out-of-set counters because the catalog has no +authoritative sets for foreign tool names. + +### Mispicks + +Every Plane call on a catalogued surface is classified by tool name as `optimal`, +`alternate`, or `out_of_set`. The task owns disjoint optimal and alternate sets, including +surface-specific overlays where present. The headline mispick rate is: + +```text +(alternate calls + out-of-set calls) / all Plane calls ``` -- `--surface` is recorded in the JSONL and (for now) only `full` is implemented; it is the - future hook for tag-filtered/transformed variants. Unknown values error. -- `--reps N` repeats each task N times (fresh seed + fresh server per rep). -- Per task-rep flow: seed → run agent → verify → append JSONL row → teardown (teardown in - a `finally`; on crash, print the orphaned project name). - -One JSONL row per task-rep: - -```json -{"run_id": "...", "ts": "...", "git_sha": "...", "surface": "full", - "model": "claude-sonnet-5", "task_id": "W1", "rep": 0, - "success": true, "verify_note": "...", "skipped": null, "error": null, - "stop_reason": "end_turn", "hit_max_iterations": false, - "calls": [{"tool": "list_projects", "class": "optimal", "args_chars": 42, - "result_tokens": 830, "result_chars": 3120, "result_kind": "text", - "is_error": false}], - "num_calls": 4, "errored_calls": 0, "alternate_calls": 0, "out_of_set_calls": 0, - "total_result_tokens": 2210, - "usage_per_iteration": [{"in": 38210, "out": 412, "cache_read": 36100, "cache_write": 0}], - "cum_input_tokens": 152840, "wall_time_s": 31.4} +`is_error` is independent of that classification. A valid call can still be an avoidable +pick, and an optimal tool can return an error. The ordered call records are retained in the +JSONL so a run can be audited after aggregation. + +### Response-token cost + +Every driver reports `result_chars` and `result_tokens` per Plane call. The character count +comes from the serialized result text actually observed by the harness. Token counts carry +an explicit provenance: + +- The API driver may use a backend token counter. If none is available or it fails, it uses + the shared deterministic character estimate. +- CLI drivers estimate from the proxy-recorded character count by default. +- With `--record-result-payloads`, CLI sidecars also retain the result text. The parent + harness uses `tiktoken` with `cl100k_base` when importable and otherwise falls back to the + same estimate. + +The estimate is `ceil(result_chars / 4)` for non-empty results. Rows and calls record +whether their values are measured, estimated, or mixed, and the report marks estimated and +mixed columns. An estimate is never presented as a measured tokenizer count. + +Payload recording is off by default because tool results contain live workspace data and +make sidecars larger. The character-derived estimate remains useful for surface comparison +because it is deterministic and monotonic in the recorded response size. + +Provider usage is a different measurement: where the driver supplies it, the harness keeps +input, output, cache-read, and cache-creation usage. Tool-result sizing describes one source +of context growth; it is not substituted for the provider's conversation-level usage. + +## Why calls are recorded at the transport boundary + +An agent's final answer is not a reliable call log. It may omit a failed lookup, summarize +several calls as one action, or claim an action it did not perform. Call-count and mispick +metrics therefore come from execution evidence. + +The API driver owns the MCP session and records each call it executes. The four CLI drivers +put `evals.proxy` between the CLI and the stdio MCP server. The proxy relays JSON-RPC bytes +without reserializing them, pairs `tools/call` requests and responses by JSON-RPC ID, and +records a request sequence on each sidecar row. The sidecar loader restores request order. +A complete proxy sidecar is authoritative; CLI event or transcript parsing is retained as +a fallback when the sidecar is incomplete. Neither source depends on the agent describing +its own behavior. + +The proxy remains standard-library-only because it runs inside the server process tree with +a scrubbed `PYTHONPATH`. It records response payloads only when explicitly requested. +Tokenization and row mapping happen later in the parent harness, where optional dependencies +are safe to import. + +## Driver and backend boundaries + +All five driver implementations satisfy `AgentDriver.run_task(...) -> AgentRun`: + +- `ApiDriver` +- `ClaudeCliDriver` +- `CodexCliDriver` +- `AntigravityCliDriver` +- `OpencodeCliDriver` + +`sdk` is a legacy CLI alias for `ApiDriver`, not a sixth implementation. The runner has one +path for all drivers: it supplies a prompt and MCP environment, receives a normalized +`AgentRun`, maps it to the common row shape, and invokes the task verifier. + +The API implementation has one further seam. `ApiDriver` owns provider-independent policy: +the stdio MCP session, tool execution loop, iteration budget, timing, result recording, +call-ID pairing, and usage accumulation. A `ModelBackend` owns provider conversation state +and wire format through three operations: + +```text +start(system, prompt, tools) +next_turn() -> Turn +add_tool_results(results) ``` -## Report (`report.py`) +This is the narrowest boundary that keeps provider-specific message roles, content blocks, +tool schemas, usage objects, and stop reasons out of the loop. `AnthropicBackend` translates +to the stable Messages API. `OpenAIBackend` translates to Chat Completions function tools +and imports the optional OpenAI SDK only when no client was injected. Both return neutral +turns containing text, tool calls, normalized usage, and a stop reason. + +CLI agents already own their model conversation and tool loop, so they implement +`AgentDriver` directly rather than pretending to be `ModelBackend` implementations. Their +subprocess, configuration, transcript, and usage differences stay within their driver +modules. + +Several loop rules are deliberately centralized in `ApiDriver`: + +- Tool results are paired to model calls by call ID, never by list position. Missing, + duplicate, or unknown IDs set `result_pair_mismatch`. +- A refusal-terminated turn records any included calls for audit but executes none of them. +- `hit_max_iterations` is set only when the iteration budget is exhausted while more tool + work remains, not merely because a valid final response used the last iteration. +- `wall_time_s` covers the model/tool loop after `list_tools`; server startup, teardown, and + post-loop token counting are outside it. +- The row records both the requested model and the provider-reported model that actually ran + when the provider returns one. + +## Task and run lifecycle +The task catalog uses plain dictionaries. Each task stays beside its verifier in the module +for its task class. The catalog package assembles those lists in a pinned historical order, +builds `TASKS_BY_ID`, and computes the battery fingerprint. + +For each task repetition, the runner creates a fresh project and only the fixture groups +declared by that task. The live sequence is: + +```text +seed -> drive -> verify -> teardown -> append row ``` -python -m evals.report evals/results/A.jsonl [evals/results/B.jsonl] + +The row is assembled as the task progresses; teardown runs in `finally` before that row is +appended. Workspace-scoped fixture objects are tracked separately from the project. A fresh +stdio server is launched for each driven task. The server environment is built from `PATH`, +`HOME`, the three Plane connection values, the selected built-in surface label when +applicable, and explicit `--server-env` additions; unrelated parent environment variables +are not inherited. + +The first line of a new result file is a meta row containing the run identity, surface, +battery, requested model, driver, provider, and Git SHA. Resume checks those identities, +skips completed task/repetition keys, and reruns rows that contain recorded errors. Result +rows preserve the common fields consumed by `evals.report` and existing JSONL readers. + +## Module layout + +```text +evals/ + cli.py argparse, command dispatch, and model-alias resolution + runner.py live lifecycle, row assembly, resume/meta handling, and canary + run.py compatibility entry point for python -m evals.run + tasks/ + __init__.py ordered catalog assembly and public task API + common.py prompt binding, matchers, and shared API lookups + read.py R1-R7 tasks and verifiers + write.py W1-W10 tasks and verifiers + schema.py S1-S5 tasks and verifiers + cross.py C1-C2 tasks and verifiers + debias.py I1-I5 and L1-L5 tasks and verifiers + drivers/ + __init__.py public exports and driver registry + base.py AgentDriver, AgentRun, normalization, and common row mapping + claude.py Claude Code CLI driver + codex.py Codex CLI driver + antigravity.py Antigravity CLI driver + opencode.py opencode CLI driver + process.py shared subprocess lifecycle + sidecar.py recording-proxy command and sidecar handling + api/ + backend.py neutral backend protocol and turn/tool dataclasses + driver.py provider-neutral MCP/model loop + anthropic.py Anthropic Messages translation + openai.py OpenAI Chat Completions translation + proxy.py stdlib-only JSON-RPC recording relay + seed.py Plane fixture creation and teardown + report.py summaries, A/B comparison, and multi-surface tables + token_counting.py shared estimate and optional local tokenizer counting ``` -Single file, per task: `n`, success as `k/n` with a 95% Wilson interval, median calls -(with IQR) vs optimal, mispick rate (alternate + out_of_set over total calls), errored -calls, capped runs (`hit_max_iterations` or `stop_reason == "max_tokens"`) and harness-error -rows (`error != null`) each reported as their own column — never silently folded into -failures, and error rows excluded from success/medians entirely — median & p95 result_tokens per -call, and median `cum_input_tokens`. - -Two files: same table with per-task deltas (B − A). **Refuse the delta mode (exit with a -message) when either file has n < 5 for any shared task** — comparative claims below that -floor are noise. Plain text, stdlib only. - -Optimal-path caveats discovered during review (bake into the sets and a comment): -- **R1**: `search_work_items` returns no state field (`WorkItemSearchItem` has only - name/id/sequence_id/identifiers). The true 1-call path is `list_work_items` (WorkItem - carries `state: str | StateLite`; `expand=state` yields the name). search is alternate. -- **S1**: `create_work_item_property` accepts inline `options`, so the optimal path is - 3 calls (`list_projects` → `resolve_work_item_type` → `create_work_item_property`) — - separate option-creation / list_work_item_types calls are alternates, not optimal. - -## Full task list (target: 20 tasks) - -Defined in the consolidation analysis; implement incrementally. IDs are stable. - -| id | prompt (abbrev) | probes | optimal | -|---|---|---|---| -| R1 | state of item titled 'Payment webhook drops retries' | list vs search (search has no state) | 1 (`list_work_items`) | -| R2 | how many urgent open items | count/list/search pick, UUID dance | 1–2 | -| R3 | items assigned to me due this week | assignee resolution | 2 | -| R4 | what's in the active cycle, anything overdue | PQL activeCycle() discovery | 1–2 | -| R5 | summarize discussion on a known item | sub-object fan-out | 2 | -| R6 | which project has more open bugs (needs 2nd project) | cross-project composition | 2–3 | -| W1 | file a bug w/ priority+assignee+label | lookup overhead before create | 3–4 | -| W2 | move item to Done | state name→UUID | 2–3 | -| W3 | comment on an item | happy path baseline | 2 | -| W4 | rename label triage→needs-triage | update_label pick | 2 | -| W5 | archive all completed items in module | no-bulk N-call burn | 2+N | -| W6 | move unfinished items Sprint 12→13, close Sprint 12 | transfer+complete workflow | 3–4 | -| W7 | mark A blocking B + add reference URL | relations-vs-links confusion | 3 | -| W8 | log 2h on an item for yesterday | worklog | 2 | -| S1 | add Severity dropdown (Critical/Major/Minor) to Bug type | property + inline options | 3 | -| S2 | add Fibonacci estimate scale, set item to 5 pts | estimates chain | 4–5 | -| S3 | create type Incident w/ required text property | type+property+attach | 4–5 | -| S4 | triage intake: accept billing, reject spam | workflow vs endpoint tools | 3–4 | -| C1 | create customer Acme, link request to item | customers domain | 3–4 | -| C2 | what shipped in release 1.2.0 | releases/changelog | 1–2 | - -## Phase 1 scope (walking skeleton) - -Implement end to end, nothing more: - -1. `tasks.py` with **R1, W1, S1 only** (verifiers included). -2. `seed.py` covering fixture groups `items`, `labels`, `bug_type`. -3. `run.py` with `--list`, `--dry-run`, and the full live path (seed→run→verify→teardown). -4. `report.py` single-file mode (A/B delta mode can be a stub that errors clearly). -5. `pyproject.toml` evals extra + `evals/results/` gitignored. - -Constraints: -- Python 3.10+, ruff clean (`ruff format evals/ && ruff check evals/` — line length 120, - rules E,F,I,UP,B per pyproject). -- Match the codebase's existing style; no classes where dicts do, no framework. -- **No live credentials exist in this checkout** — done means: `--list` and `--dry-run` - work without network, imports resolve in a fresh venv after - `uv pip install -e ".[dev,evals]"`, ruff passes. The live path must be complete and - plausible but cannot be executed yet. -- Do NOT commit. Do NOT touch `.ccwrc`, `.env*`, or anything under `plane_mcp/`. +The stable import and command surfaces are intentional: `from evals.tasks import ...`, +`from evals.drivers import ...`, and `python -m evals.run` remain the public boundaries even +though their implementations are split across packages and focused modules. diff --git a/evals/README.md b/evals/README.md index 07587bd5..96ce575d 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,20 +1,23 @@ # Eval harness — runbook Measures how well an LLM agent completes real Plane tasks through an MCP tool surface. -Every task runs against a **live** Plane API with fixtures seeded and torn down per run, -and is graded by a verifier that reads the API back — not by inspecting the agent's prose. +Every live task repetition uses a **live** Plane API with its own seeded fixtures and +teardown. Mutation tasks are verified by reading Plane back; read tasks match the final +answer against facts from the seed context or the API rather than trusting the agent's +claim that it succeeded. The harness is agent-agnostic and surface-agnostic: any stdio MCP server can be measured -(`--server-cmd`), driven by any of five agent backends. `DESIGN.md` explains why it is +(`--server-cmd`), driven by any of five driver implementations. `DESIGN.md` explains why it is built this way; this file is how to run it. What you get per task: pass/fail, tool calls to done, which tools were picked (and whether -they were the optimal ones), errors, and the agent's final text. +they were the optimal ones), response size and token-count provenance, errors, and the +agent's final text. ## Prerequisites -1. **A local Plane API.** `evals/env.sh` starts plane-ee on `:8000` plus a mock - feature-flag server on `:9911` that turns every flag on. +1. **A Plane API endpoint.** For local runs, `evals/env.sh` starts plane-ee on `:8000` plus + a mock feature-flag server on `:9911` that turns every discovered flag on. ```bash export PLANE_EE_API_DIR=/path/to/plane-ee/apps/api @@ -22,14 +25,14 @@ they were the optimal ones), errors, and the agent's final text. evals/env.sh up # down | status ``` -2. **A workspace and an API key** on that instance, with a Business/Enterprise license so - the gated fixtures (customers, releases) can be seeded. +2. **A workspace and an API key** on that instance. The key must be able to create and + delete the catalog's project and workspace-scoped fixtures. Genuine plan or feature + gates are recorded as skips where the fixture code handles them. ```bash export EVAL_PLANE_BASE_URL=http://localhost:8000 export EVAL_PLANE_WORKSPACE_SLUG= export EVAL_PLANE_API_KEY=plane_api_... - unset REDIS_HOST REDIS_PORT # else the SDK client picks up a stale cache config ``` 3. **Model access for the driver you pick.** The API driver uses @@ -43,34 +46,35 @@ they were the optimal ones), errors, and the agent's final text. .venv/bin/python -m evals.run --driver api --provider anthropic --model sonnet \ --surface full --out results/api.jsonl -# Everything, one surface -.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ +# Everything, one surface (free-form model IDs pass through to the CLI) +.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ --surface full --out results/legacy.jsonl # A few tasks while iterating -.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ +.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ --surface full --tasks W5,W8 --out results/spot.jsonl # Someone else's server (a PR branch, another repo) — "external mode" -.venv/bin/python -m evals.run --driver codex-cli --model gpt-5.6-sol \ +.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ --surface their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ --server-env PLANE_MCP_TOOLS_VERSION=v2 --out results/their-pr.jsonl ``` -Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed -`(task, rep)` pairs, retry only infra failures), `--list` / `--dry-run` (no network). +Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed or +skipped `(task, rep)` pairs and retry rows with recorded errors), `--list` / `--dry-run` +(no network). -**External mode** (`--server-cmd`) runs every task with no surface-based skips, and turns -off mispick classification — foreign tool names have no optimal/alternate sets to score -against, so call *counts* stay comparable but "mispicks" reads `n/a`. +**External mode** (`--server-cmd`) runs every task with no surface-based skips and records +`classification: "external"`. Foreign tool names have no catalogued optimal/alternate sets, +so use success, call counts, and errors for those rows; their mispick values are not +comparable to catalogued surfaces. -**On surfaces.** `--surface` without `--server-cmd` runs this repo's own server, and passes -the surface through as `PLANE_MCP_SURFACE`. This branch's server serves one surface, so -only `full` is real here: `v2` / `v2-schema` set an env var nothing reads, and you would -get legacy results labelled as something else. The task catalog keeps its per-surface -overlays (`surface_tools`) so those surfaces score correctly if the server ever grows them. -**Measure any other surface through `--server-cmd`** — that is how the PR surfaces were -compared, and it is honest about what it ran because it launches the server you name. +**On surfaces.** Without `--server-cmd`, `full` launches this repo's server with no surface +variable. The `v2` and `v2-schema` labels set `PLANE_MCP_SURFACE`, and the task catalog has +matching `surface_tools` overlays. This tree's `plane_mcp` server does not read that variable, +so selecting either label here would run the same server under a misleading label. Use +`--server-cmd` to launch a server that actually implements another surface; external rows +intentionally do not use this catalog's tool-choice classification. ### Drivers @@ -78,13 +82,14 @@ compared, and it is honest about what it ran because it launches the server you |---|---|---| | `api` | Owned API + MCP loop | Provider-neutral; `--provider anthropic` (default) or `openai` | | `sdk` | Alias for `api` | Retained for old commands/result pipelines | -| `codex-cli` | OpenAI Codex CLI | Pass a real model id (`gpt-5.6-sol`); the short-alias table is incomplete | +| `codex-cli` | OpenAI Codex CLI | **Pass a real model id.** The harness forwards `sonnet` / `haiku` unchanged and Codex rejects them, so the short aliases fail on this driver | | `claude-cli` | Claude Code CLI | `--model sonnet` / `haiku` | | `antigravity-cli` | Antigravity CLI (`agy`) | Runs under a synthetic HOME so its MCP config is ours, not yours | | `opencode-cli` | opencode | Temp project config per run | Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool -calls are counted from the wire rather than from whatever the agent claims it did. +calls are normally counted from the wire rather than from whatever the agent claims it did. +If a sidecar is incomplete, the driver can fall back to its CLI event stream or transcript. The API driver executes MCP calls itself, records exact result character counts, and sizes result tokens without making a provider request per result. A backend may supply a token @@ -111,8 +116,9 @@ habit; use it only when the more sensitive, larger sidecar is justified. Rows are deduped latest-wins per `(task_id, rep, surface)`, so a re-run of a single task supersedes its earlier row in the same file. Skipped tasks are excluded from success -denominators — a surface that cannot do something is not punished as a failure, it is -reported as a skip. +denominators, as are rows with recorded errors. Result-token columns use `~` for estimates, +`*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not +recorded. ## Running surfaces in parallel @@ -130,7 +136,18 @@ in one workspace but not another skew every workspace-wide task in that column. ## Adding a task -Tasks live in `evals/tasks.py`. A task is a dict: +Tasks live in the `evals/tasks/` package, grouped by task class and kept beside their +verifiers: + +- `read.py`: R1-R7 +- `write.py`: W1-W10 +- `schema.py`: S1-S5 +- `cross.py`: C1-C2 +- `debias.py`: I1-I5 and L1-L5 + +Shared prompt, matcher, and API lookup machinery lives in `common.py`. `tasks/__init__.py` +assembles the class lists in the pinned catalog order and re-exports the public task API. +A task is a dict: ```python { @@ -140,7 +157,12 @@ Tasks live in `evals/tasks.py`. A task is a dict: "optimal_calls": 3, "optimal_tools": {"list_cycles", "complete_cycle"}, # scored as optimal picks "alternate_tools": {"list_projects"}, # acceptable, not optimal - "surface_tools": {"v2": {"optimal_tools": {"close_cycle"}}}, # per-surface overlay + "surface_tools": { + "v2": { + "optimal_tools": {"close_cycle"}, + "alternate_tools": {"list_cycles"}, + } + }, "needs": {"items", "cycles"}, # fixtures to seed "verify": verify_w11, } @@ -156,13 +178,14 @@ genuinely cannot do the task. That is reported as a capability gap, not a failur ### Writing a verifier -Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)` and must read -state back through the API. Two rules, both learned the hard way: +Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)`. Keep a new task +and its verifier in the same class module, add it to that module's exported task list, and +preserve the assembly order in `tasks/__init__.py`. -**Never parse natural language.** Constrain the output in the prompt instead — end the -prompt with `Answer with a line 'count: N'` and match that line exactly. Regex over prose -produces both false passes ("10 attachments" satisfying a truth of 0) and false failures -("three" for 3), and no amount of tuning fixes it. +Mutation verifiers must read the resulting state through the Plane API. Read verifiers must +derive the expected facts from the API or seed context and match an explicit answer contract +or exact seeded values. For numeric answers, prefer a prompt such as `Answer with a line +'count: N'` and the shared contract matcher; a loose substring can make `4` match `24`. **Check the shape the API actually returns.** Dates come back as timestamps (`2026-08-12T00:00:00Z`), so comparing one to a bare `2026-08-12` silently never matches — @@ -175,24 +198,29 @@ Then prove the verifier can fail: .venv/bin/python -m evals.run --canary --surface full ``` -The canary seeds every task, calls each verifier with an **empty** agent result, and exits -non-zero if any verifier passes a do-nothing agent. Run it after touching tasks, fixtures, -or verifiers. +The canary seeds each surface-eligible task, calls its verifier with an **empty** agent +result, and exits non-zero if any verifier passes a do-nothing agent. Run it after touching +tasks, fixtures, or verifiers. -**Make the task achievable before blaming a surface.** A fixture that forbids what the -prompt asks turns the task into a coin flip on tool choice and will implicate whichever -server happens to pick differently. W6 was pre-closing a cycle the agent was asked to -close; it took two full runs to notice, because a side effect of an unrelated call was -tripping the verifier. +**Make the task achievable before blaming a surface.** For example, W6 declares the +`cycles_open_past` fixture variant because it asks the agent to close Sprint 12; the seeder +must not pre-close the cycle that the task is meant to change. ## Local gotchas -- **Comment activity never appears** without a running activity worker, so the tasks that - read the activity feed self-skip (`env:no-activity-worker`) rather than fail. -- **A feature-flag cache poisoned by the wrong disco.** Anything that talks to the DB while - sourcing `plane-ee/apps/api/.env` uses the *remote* flag server (all flags off) and - caches that answer for a workspace the API server would otherwise serve from the local - mock. Symptom: gated endpoints 402 and the canary reports every verifier broken. Fix: - clear `ff::*` and rotate `ff_ver:` (`plane.payment.flags.cache`). +- If seeded comments do not materialize as activities, the activity-feed task self-skips + with `env:no-activity-worker` rather than failing the agent. +- **Gated endpoints returning 402 on a workspace that should work.** Feature flags are + cached per workspace, and the cache does not record which flag server answered. Any + process that touches the DB while pointed at a *different* flag server than the running + API — sourcing `plane-ee/apps/api/.env` gets you the remote one, where these flags are + off — caches that answer for the workspace, and the API then serves the cached miss + instead of asking its own mock. The tell is a canary that reports every verifier broken + at once. Clear `ff::*` and rotate `ff_ver:` (`plane.payment.flags.cache`) + and the next request refetches. Observed while creating a workspace from a Django + shell; a plan/licence problem looks identical from the outside, so check this first. +- A workspace licence is **not** required for local runs: the mock flag server enables + every flag regardless, and an unlicensed workspace seeds all fixtures (verified by + canary against a workspace with no licence row). - **Offline tests** cover the harness itself and need no Plane instance: `env -u REDIS_HOST -u REDIS_PORT .venv/bin/python -m pytest -q --ignore=tests/test_integration.py` From ea50b74a16fb7fb2f30bd7cc5a8dd3d84945db3b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 09:50:01 +0530 Subject: [PATCH 08/93] Make the API driver provider-neutral and the model tiers vendor-free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver called itself generic while being Anthropic-shaped: it imported both concrete backends and picked between them with an if/else, and the loop branched on Anthropic's own stop_reason strings ("refusal", "pause_turn"), so any other provider had to impersonate Anthropic to work. Adding a third provider meant editing the supposedly generic driver. Stop reasons are now an enum we own, with each backend mapping its provider's values in, and backends register themselves so the driver imports none of them — a test registers a dummy backend and drives the whole loop through it to prove a new provider needs no driver changes. The enum's serialized values match what rows already carried, so old result files stay comparable, and each turn keeps the provider's raw reason for debugging. Model tiers were the same leak one level up: sonnet and haiku were the harness vocabulary, so every non-Anthropic driver mapped from a vendor it has nothing to do with, and codex-cli forwarded "sonnet" to a CLI that rejects it. The tiers are now standard and fast, resolved per driver and per provider, with every mapping verified against the provider rather than assumed. Anything that is not a tier passes through untouched. OpenCode's tiers are deliberately unmapped because its models are project-configured: it fails with instructions instead of guessing, since a plausible wrong ID silently benchmarks a different model. Co-Authored-By: Claude Fable 5 --- evals/DESIGN.md | 8 +- evals/README.md | 36 +++++- evals/cli.py | 108 +++++++++++------ evals/drivers/api/__init__.py | 40 ++++++- evals/drivers/api/anthropic.py | 53 +++++++-- evals/drivers/api/backend.py | 212 ++++++++++++++++++++++++++++++++- evals/drivers/api/driver.py | 66 ++++++---- evals/drivers/api/openai.py | 58 ++++++--- evals/drivers/base.py | 4 + evals/run.py | 25 ++-- evals/runner.py | 37 +++++- tests/test_evals_api_driver.py | 199 +++++++++++++++++++++++++++---- tests/test_evals_drivers.py | 1 + tests/test_evals_hardening.py | 34 +++++- tests/test_evals_proxy.py | 70 +++++++++-- 15 files changed, 800 insertions(+), 151 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index d8c45f97..f5813a19 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -161,8 +161,8 @@ Several loop rules are deliberately centralized in `ApiDriver`: work remains, not merely because a valid final response used the last iteration. - `wall_time_s` covers the model/tool loop after `list_tools`; server startup, teardown, and post-loop token counting are outside it. -- The row records both the requested model and the provider-reported model that actually ran - when the provider returns one. +- The row records the requested model token, requested tier (when present), resolved model ID, + and the provider-reported model that actually ran when the provider returns one. ## Task and run lifecycle @@ -185,7 +185,7 @@ applicable, and explicit `--server-env` additions; unrelated parent environment are not inherited. The first line of a new result file is a meta row containing the run identity, surface, -battery, requested model, driver, provider, and Git SHA. Resume checks those identities, +battery, requested model/tier, resolved model, driver, provider, and Git SHA. Resume checks those identities, skips completed task/repetition keys, and reruns rows that contain recorded errors. Result rows preserve the common fields consumed by `evals.report` and existing JSONL readers. @@ -193,7 +193,7 @@ rows preserve the common fields consumed by `evals.report` and existing JSONL re ```text evals/ - cli.py argparse, command dispatch, and model-alias resolution + cli.py argparse, command dispatch, and model-tier resolution runner.py live lifecycle, row assembly, resume/meta handling, and canary run.py compatibility entry point for python -m evals.run tasks/ diff --git a/evals/README.md b/evals/README.md index 96ce575d..99506991 100644 --- a/evals/README.md +++ b/evals/README.md @@ -43,7 +43,7 @@ agent's final text. ```bash # Provider-neutral API loop (default provider: Anthropic) -.venv/bin/python -m evals.run --driver api --provider anthropic --model sonnet \ +.venv/bin/python -m evals.run --driver api --provider anthropic --model standard \ --surface full --out results/api.jsonl # Everything, one surface (free-form model IDs pass through to the CLI) @@ -80,12 +80,36 @@ intentionally do not use this catalog's tool-choice classification. | Driver | Backend | Notes | |---|---|---| -| `api` | Owned API + MCP loop | Provider-neutral; `--provider anthropic` (default) or `openai` | +| `api` | Owned API + MCP loop | Provider-neutral; tiers resolve for `--provider anthropic` (default) or `openai` | | `sdk` | Alias for `api` | Retained for old commands/result pipelines | -| `codex-cli` | OpenAI Codex CLI | **Pass a real model id.** The harness forwards `sonnet` / `haiku` unchanged and Codex rejects them, so the short aliases fail on this driver | -| `claude-cli` | Claude Code CLI | `--model sonnet` / `haiku` | -| `antigravity-cli` | Antigravity CLI (`agy`) | Runs under a synthetic HOME so its MCP config is ours, not yours | -| `opencode-cli` | opencode | Temp project config per run | +| `codex-cli` | OpenAI Codex CLI | `standard` and `fast` resolve to verified GPT-5.6 IDs | +| `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku` | +| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; runs under a synthetic HOME so its MCP config is ours, not yours | +| `opencode-cli` | OpenCode | Tiers are intentionally unmapped; pass an explicit ID listed by `opencode models` | + +### Model tiers + +The harness has exactly two provider-neutral tiers: `standard`, the workhorse used for the +battery, and `fast`, the lower-cost option. Resolution is scoped to both driver and provider: + +| Driver | Provider | `standard` | `fast` | +|---|---|---|---| +| `api` / `sdk` | Anthropic | `claude-sonnet-5` | `claude-haiku-4-5` | +| `api` / `sdk` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | +| `claude-cli` | Anthropic | `sonnet` | `haiku` | +| `codex-cli` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | +| `antigravity-cli` | Google | `gemini-3.6-flash-high` | `gemini-3.6-flash-low` | +| `opencode-cli` | Project-configured | **unmapped** | **unmapped** | + +Only `standard` and `fast` are harness vocabulary. Every other `--model` value passes through +unchanged, including vendor aliases such as `sonnet` / `haiku` and full IDs such as +`claude-opus-5`, `gpt-5.6-sol`, or `openai/gpt-5.6-sol`. An unmapped tier fails before the run +and tells you to pass an explicit model ID; the harness never guesses one. + +Result and meta rows keep `requested_model`, `requested_tier`, and `resolved_model` separately. +The compatibility field `model` remains the provider-reported model when available, otherwise +the resolved ID. This makes old readers continue to work while preserving which tier produced +the row even after its mapping changes. Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool calls are normally counted from the wire rather than from whatever the agent claims it did. diff --git a/evals/cli.py b/evals/cli.py index a9a69e1c..3b06c890 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -15,51 +15,78 @@ from typing import Any from evals.drivers import KNOWN_DRIVERS +from evals.drivers.api import ( + KNOWN_API_PROVIDERS, + MODEL_TIERS, + UnmappedModelTierError, + backend_model_aliases, + resolve_backend_model, +) from evals.runner import KNOWN_SURFACES, run_canary, run_live from evals.seed import seed_plan from evals.tasks import TASKS, format_task_prompt, get_tasks -MODEL_ALIASES: dict[str, str] = { - "sonnet": "claude-sonnet-5", - "haiku": "claude-haiku-4-5", +API_MODEL_TIERS: dict[str, dict[str, str]] = { + provider: aliases for provider in KNOWN_API_PROVIDERS if (aliases := backend_model_aliases(provider)) } -API_MODEL_ALIASES: dict[str, dict[str, str]] = { - "anthropic": MODEL_ALIASES, - # Preserve the harness's representative/fast intent when the user switches - # providers without also overriding the historical sonnet/haiku aliases. - "openai": {"sonnet": "gpt-5", "haiku": "gpt-5-mini"}, +# CLI drivers have an implicit provider selected by their own authentication +# and configuration. Keep the provider dimension explicit so a tier never +# crosses vendor boundaries by accident. +CLI_DRIVER_PROVIDERS: dict[str, str | None] = { + "claude-cli": "anthropic", + "codex-cli": "openai", + "antigravity-cli": "google", + # OpenCode is multi-provider and location-configured. Its installed catalog + # is the only reliable source, so the harness does not guess a default. + "opencode-cli": None, } -# Per-driver resolution of the short harness aliases (sonnet/haiku). -# Drivers that need provider/model form get qualified defaults; unknown -# strings (e.g. ``anthropic/claude-…``) pass through unchanged. -CLI_MODEL_ALIASES: dict[str, dict[str, str]] = { - "claude-cli": {"sonnet": "sonnet", "haiku": "haiku"}, - "codex-cli": {"sonnet": "sonnet", "haiku": "haiku"}, - "antigravity-cli": { - "sonnet": "gemini-3.6-flash-high", - "haiku": "gemini-3.6-flash-low", +CLI_MODEL_TIERS: dict[str, dict[str, dict[str, str]]] = { + "claude-cli": { + "anthropic": {"standard": "sonnet", "fast": "haiku"}, + }, + "codex-cli": { + "openai": backend_model_aliases("openai"), }, - "opencode-cli": { - "sonnet": "anthropic/claude-sonnet-4-20250514", - "haiku": "anthropic/claude-haiku-4-5-20251001", + "antigravity-cli": { + "google": { + "standard": "gemini-3.6-flash-high", + "fast": "gemini-3.6-flash-low", + }, }, + "opencode-cli": {}, } DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "results" -def resolve_model_for_driver(driver_name: str, model: str, *, provider: str = "anthropic") -> str: - """Map a harness model token to the string the given driver expects. +def resolve_model_for_driver(driver_name: str, model: str, *, provider: str | None = None) -> str: + """Resolve a harness tier for a driver/provider, or pass a model ID through. - Known short aliases (sonnet/haiku) are looked up per-driver. Any other - string (including already-qualified ``provider/model``) is passed through. + Only ``standard`` and ``fast`` are tier names. Any other string, including + vendor aliases and qualified provider/model IDs, is passed through exactly. """ key = (driver_name or "api").strip().lower() if key in ("api", "sdk"): - table = API_MODEL_ALIASES.get(provider.strip().lower()) or {} - return table.get(model, model) - table = CLI_MODEL_ALIASES.get(key) or {} - return table.get(model, model) + return resolve_backend_model(provider or "anthropic", model) + if model not in MODEL_TIERS: + return model + if key not in CLI_DRIVER_PROVIDERS: + raise ValueError(f"unknown driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}") + provider_id = provider.strip().lower() if provider else CLI_DRIVER_PROVIDERS[key] + if provider_id is None: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for driver {key!r}; OpenCode models depend on " + "the providers configured for this project. Pass an explicit provider/model ID with " + "--model, using one listed by 'opencode models'" + ) + table = CLI_MODEL_TIERS.get(key, {}).get(provider_id, {}) + try: + return table[model] + except KeyError as exc: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for driver {key!r} and provider {provider_id!r}; " + "pass an explicit model ID with --model" + ) from exc def parse_args(argv: list[str] | None = None) -> argparse.Namespace: @@ -70,10 +97,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p.add_argument( "--model", type=str, - default="sonnet", + default="standard", help=( - "Model alias (sonnet/haiku) or a free-form provider/model id. " - "Short aliases are remapped per --driver (opencode/antigravity get qualified names)." + "Harness tier (standard/fast) or a free-form model ID. " + "Tiers resolve per driver and provider; all other strings pass through unchanged." ), ) p.add_argument("--reps", type=int, default=1, help="Repetitions per task") @@ -118,7 +145,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--provider", type=str, default="anthropic", - choices=("anthropic", "openai"), + choices=sorted(KNOWN_API_PROVIDERS), help="Model API provider for --driver api/sdk (default: anthropic).", ) p.add_argument( @@ -248,7 +275,15 @@ def main(argv: list[str] | None = None) -> int: else: out = DEFAULT_OUT_DIR / f"{uuid.uuid4().hex}.jsonl" - model_id = resolve_model_for_driver(driver_name, args.model, provider=args.provider) + try: + model_id = resolve_model_for_driver( + driver_name, + args.model, + provider=args.provider if driver_name in ("api", "sdk") else None, + ) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 return asyncio.run( run_live( tasks, @@ -268,10 +303,11 @@ def main(argv: list[str] | None = None) -> int: __all__ = [ - "API_MODEL_ALIASES", - "CLI_MODEL_ALIASES", + "API_MODEL_TIERS", + "CLI_DRIVER_PROVIDERS", + "CLI_MODEL_TIERS", "DEFAULT_OUT_DIR", - "MODEL_ALIASES", + "MODEL_TIERS", "cmd_dry_run", "cmd_list", "main", diff --git a/evals/drivers/api/__init__.py b/evals/drivers/api/__init__.py index a3e88491..fef57405 100644 --- a/evals/drivers/api/__init__.py +++ b/evals/drivers/api/__init__.py @@ -1,17 +1,53 @@ -"""Provider-generic API driver and backend translations.""" +"""Provider-generic API driver and registered backend translations.""" +# Import built-in adapters for their registrations. Each adapter owns its SDK +# translation and keeps its optional SDK import lazy until construction. from evals.drivers.api.anthropic import AnthropicBackend -from evals.drivers.api.backend import ModelBackend, ToolCall, ToolResult, ToolSpec, Turn +from evals.drivers.api.backend import ( + BACKEND_REGISTRY, + KNOWN_API_PROVIDERS, + MODEL_TIERS, + BackendFactory, + BackendRegistration, + BackendRegistry, + ModelBackend, + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + UnmappedModelTierError, + Usage, + backend_model_aliases, + create_backend, + register_backend, + resolve_backend_model, + unregister_backend, +) from evals.drivers.api.driver import ApiDriver from evals.drivers.api.openai import OpenAIBackend __all__ = [ + "BACKEND_REGISTRY", + "KNOWN_API_PROVIDERS", + "MODEL_TIERS", "AnthropicBackend", "ApiDriver", + "BackendFactory", + "BackendRegistration", + "BackendRegistry", "ModelBackend", "OpenAIBackend", + "StopReason", "ToolCall", "ToolResult", "ToolSpec", "Turn", + "UnmappedModelTierError", + "Usage", + "backend_model_aliases", + "create_backend", + "register_backend", + "resolve_backend_model", + "unregister_backend", ] diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py index aa6143b6..ee137c01 100644 --- a/evals/drivers/api/anthropic.py +++ b/evals/drivers/api/anthropic.py @@ -4,7 +4,15 @@ from typing import Any -from evals.drivers.api.backend import ToolCall, ToolResult, ToolSpec, Turn +from evals.drivers.api.backend import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + Usage, + register_backend, +) def _field(value: Any, name: str, default: Any = None) -> Any: @@ -13,15 +21,28 @@ def _field(value: Any, name: str, default: Any = None) -> Any: return getattr(value, name, default) -def _usage_dict(usage: Any) -> dict[str, int] | None: +def _normalize_usage(usage: Any) -> Usage | None: if usage is None: return None - return { - "in": int(_field(usage, "input_tokens", 0) or 0), - "out": int(_field(usage, "output_tokens", 0) or 0), - "cache_read": int(_field(usage, "cache_read_input_tokens", 0) or 0), - "cache_write": int(_field(usage, "cache_creation_input_tokens", 0) or 0), - } + return Usage( + input_tokens=int(_field(usage, "input_tokens", 0) or 0), + output_tokens=int(_field(usage, "output_tokens", 0) or 0), + cache_read_input_tokens=int(_field(usage, "cache_read_input_tokens", 0) or 0), + cache_creation_input_tokens=int(_field(usage, "cache_creation_input_tokens", 0) or 0), + ) + + +def _normalize_stop_reason(value: Any) -> tuple[StopReason, str | None]: + raw = str(value) if value is not None else None + reason = { + "end_turn": StopReason.END_TURN, + "tool_use": StopReason.TOOL_USE, + "max_tokens": StopReason.MAX_TOKENS, + "refusal": StopReason.REFUSAL, + "pause_turn": StopReason.PAUSE_TURN, + "model_context_window_exceeded": StopReason.MODEL_CONTEXT_WINDOW_EXCEEDED, + }.get(raw, StopReason.UNKNOWN) + return reason, raw class AnthropicBackend: @@ -100,11 +121,13 @@ def next_turn(self) -> Turn: response_model = _field(message, "model") if response_model: self.actual_model = str(response_model) + stop_reason, provider_stop_reason = _normalize_stop_reason(_field(message, "stop_reason")) return Turn( text="\n".join(text_parts), tool_calls=calls, - usage=_usage_dict(_field(message, "usage")), - stop_reason=_field(message, "stop_reason"), + usage=_normalize_usage(_field(message, "usage")), + stop_reason=stop_reason, + provider_stop_reason=provider_stop_reason, ) def add_tool_results(self, results: list[ToolResult]) -> None: @@ -124,4 +147,14 @@ def add_tool_results(self, results: list[ToolResult]) -> None: ) +register_backend( + AnthropicBackend.provider, + AnthropicBackend, + model_aliases={ + "standard": "claude-sonnet-5", + "fast": "claude-haiku-4-5", + }, +) + + __all__ = ["AnthropicBackend"] diff --git a/evals/drivers/api/backend.py b/evals/drivers/api/backend.py index ebd2001a..a50b2fa5 100644 --- a/evals/drivers/api/backend.py +++ b/evals/drivers/api/backend.py @@ -1,10 +1,60 @@ -"""Provider-neutral types for API-backed eval agent loops.""" +"""Provider-neutral contracts and registry for API-backed eval loops.""" from __future__ import annotations +from collections.abc import Callable, Iterator, Mapping, Set from dataclasses import dataclass +from enum import Enum from typing import Any, Protocol +MODEL_TIERS = frozenset({"standard", "fast"}) + + +class UnmappedModelTierError(ValueError): + """Raised when a provider has no verified model for a harness tier.""" + + +class StopReason(str, Enum): + """Harness-owned reasons why a provider turn stopped. + + Values intentionally preserve the strings historically emitted by the + Anthropic API path so old and new result rows remain comparable. Provider + adapters retain the provider's original value separately on ``Turn``. + + ``END_TURN`` is a normal completed response; ``TOOL_USE`` requests tool + execution; ``MAX_TOKENS`` and ``MODEL_CONTEXT_WINDOW_EXCEEDED`` are token + limits; ``REFUSAL`` is terminal and prevents requested side effects; + ``PAUSE_TURN`` asks the loop to continue; and ``UNKNOWN`` is the explicit + fallback for missing or newly introduced provider values. + """ + + END_TURN = "end_turn" + TOOL_USE = "tool_use" + MAX_TOKENS = "max_tokens" + REFUSAL = "refusal" + PAUSE_TURN = "pause_turn" + MODEL_CONTEXT_WINDOW_EXCEEDED = "model_context_window_exceeded" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class Usage: + """Provider-neutral token usage for one model turn.""" + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_input_tokens: int = 0 + cache_creation_input_tokens: int = 0 + + def to_legacy_dict(self) -> dict[str, int]: + """Return the established per-iteration JSONL shape.""" + return { + "in": self.input_tokens, + "out": self.output_tokens, + "cache_read": self.cache_read_input_tokens, + "cache_write": self.cache_creation_input_tokens, + } + @dataclass(frozen=True) class ToolSpec: @@ -40,8 +90,9 @@ class Turn: text: str tool_calls: list[ToolCall] - usage: dict[str, int] | None - stop_reason: str | None + usage: Usage | None + stop_reason: StopReason + provider_stop_reason: str | None = None class ModelBackend(Protocol): @@ -54,6 +105,7 @@ class ModelBackend(Protocol): provider: str model: str actual_model: str + client: Any def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: ... @@ -62,4 +114,156 @@ def next_turn(self) -> Turn: ... def add_tool_results(self, results: list[ToolResult]) -> None: ... -__all__ = ["ModelBackend", "ToolCall", "ToolResult", "ToolSpec", "Turn"] +BackendFactory = Callable[..., ModelBackend] + + +@dataclass(frozen=True) +class BackendRegistration: + """A registered provider factory and the model aliases it owns.""" + + factory: BackendFactory + model_aliases: Mapping[str, str] + + +class BackendRegistry: + """Mutable registry of API providers, populated by backend modules.""" + + def __init__(self) -> None: + self.registrations: dict[str, BackendRegistration] = {} + + @staticmethod + def normalize_name(provider: str) -> str: + name = provider.strip().lower() + if not name: + raise ValueError("API provider name cannot be empty") + return name + + def register( + self, + provider: str, + factory: BackendFactory, + *, + model_aliases: Mapping[str, str] | None = None, + ) -> None: + name = self.normalize_name(provider) + if name in self.registrations: + raise ValueError(f"API provider {name!r} is already registered") + aliases = {str(alias): str(model) for alias, model in (model_aliases or {}).items()} + self.registrations[name] = BackendRegistration(factory=factory, model_aliases=aliases) + + def unregister(self, provider: str) -> None: + """Remove a provider registration, primarily for isolated tests.""" + self.registrations.pop(self.normalize_name(provider), None) + + def names(self) -> frozenset[str]: + return frozenset(self.registrations) + + def resolve(self, provider: str) -> BackendRegistration: + name = self.normalize_name(provider) + try: + return self.registrations[name] + except KeyError as exc: + raise ValueError(f"unknown API provider {name!r}; expected one of {sorted(self.registrations)}") from exc + + def create( + self, + provider: str, + model: str, + *, + max_tokens: int, + client: Any | None = None, + ) -> ModelBackend: + registration = self.resolve(provider) + return registration.factory(model, max_tokens=max_tokens, client=client) + + def resolve_model(self, provider: str, model: str) -> str: + registration = self.resolve(provider) + if model in MODEL_TIERS and model not in registration.model_aliases: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for API provider {self.normalize_name(provider)!r}; " + "pass an explicit model ID with --model" + ) + return registration.model_aliases.get(model, model) + + def model_aliases(self, provider: str) -> dict[str, str]: + return dict(self.resolve(provider).model_aliases) + + +class RegisteredProviderNames(Set[str]): + """Live set view over the providers in a ``BackendRegistry``.""" + + def __init__(self, registry: BackendRegistry) -> None: + self.registry = registry + + def __contains__(self, value: object) -> bool: + return value in self.registry.registrations + + def __iter__(self) -> Iterator[str]: + return iter(self.registry.registrations) + + def __len__(self) -> int: + return len(self.registry.registrations) + + +BACKEND_REGISTRY = BackendRegistry() +KNOWN_API_PROVIDERS: Set[str] = RegisteredProviderNames(BACKEND_REGISTRY) + + +def register_backend( + provider: str, + factory: BackendFactory, + *, + model_aliases: Mapping[str, str] | None = None, +) -> None: + """Register a provider factory and any aliases owned by that provider.""" + BACKEND_REGISTRY.register(provider, factory, model_aliases=model_aliases) + + +def unregister_backend(provider: str) -> None: + """Remove a provider registration.""" + BACKEND_REGISTRY.unregister(provider) + + +def create_backend( + provider: str, + model: str, + *, + max_tokens: int, + client: Any | None = None, +) -> ModelBackend: + """Construct the backend registered for ``provider``.""" + return BACKEND_REGISTRY.create(provider, model, max_tokens=max_tokens, client=client) + + +def resolve_backend_model(provider: str, model: str) -> str: + """Resolve only aliases declared by the selected provider.""" + return BACKEND_REGISTRY.resolve_model(provider, model) + + +def backend_model_aliases(provider: str) -> dict[str, str]: + """Return a copy of one provider's owned alias mapping.""" + return BACKEND_REGISTRY.model_aliases(provider) + + +__all__ = [ + "BACKEND_REGISTRY", + "KNOWN_API_PROVIDERS", + "MODEL_TIERS", + "BackendFactory", + "BackendRegistration", + "BackendRegistry", + "ModelBackend", + "RegisteredProviderNames", + "StopReason", + "ToolCall", + "ToolResult", + "ToolSpec", + "Turn", + "UnmappedModelTierError", + "Usage", + "backend_model_aliases", + "create_backend", + "register_backend", + "resolve_backend_model", + "unregister_backend", +] diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index eaf119f0..a248fdd7 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -15,14 +15,18 @@ from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client -from evals.drivers.api.anthropic import AnthropicBackend -from evals.drivers.api.backend import ModelBackend, ToolResult, ToolSpec -from evals.drivers.api.openai import OpenAIBackend +from evals.drivers.api.backend import ( + KNOWN_API_PROVIDERS, + ModelBackend, + StopReason, + ToolResult, + ToolSpec, + create_backend, +) from evals.drivers.base import AgentRun from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens DEFAULT_MAX_TOKENS = 8192 -KNOWN_API_PROVIDERS = frozenset({"anthropic", "openai"}) BackendFactory = Callable[[str, int], ModelBackend] McpSessionFactory = Callable[[StdioServerParameters], Any] @@ -95,7 +99,7 @@ def tool_result_from_mcp(call_id: str, raw_result: Any) -> ToolResult: class ApiDriver: - """Run an owned tool loop against Anthropic or OpenAI and a stdio MCP server.""" + """Run an owned model/tool loop through a registered API backend.""" name = "api" @@ -126,10 +130,12 @@ def __init__( def _make_backend(self, model: str) -> ModelBackend: if self.backend_factory is not None: return self.backend_factory(model, self.max_tokens) - if self.provider == "anthropic": - backend = AnthropicBackend(model, max_tokens=self.max_tokens, client=self.client) - else: - backend = OpenAIBackend(model, max_tokens=self.max_tokens, client=self.client) + backend = create_backend( + self.provider, + model, + max_tokens=self.max_tokens, + client=self.client, + ) # Delay credential-dependent client creation until the first non-skipped # task, then reuse the provider's connection pool across the battery. if self.client is None: @@ -201,11 +207,16 @@ async def _run_task( calls: list[dict[str, Any]] = [] pending_results: list[tuple[int, str]] = [] usage_per_iteration: list[dict[str, int]] = [] + total_input_tokens = 0 + total_output_tokens = 0 + total_cache_read_input_tokens = 0 + total_cache_creation_input_tokens = 0 result_pair_mismatch = False hit_max_iterations = False iterations = 0 final_text = "" - stop_reason: str | None = None + stop_reason: StopReason | None = None + provider_stop_reason: str | None = None params = self._server_params(mcp_env, cwd) async with self._mcp_session(params) as mcp_client: @@ -224,8 +235,13 @@ async def _run_task( iterations += 1 final_text = turn.text stop_reason = turn.stop_reason + provider_stop_reason = turn.provider_stop_reason if turn.usage is not None: - usage_per_iteration.append(dict(turn.usage)) + usage_per_iteration.append(turn.usage.to_legacy_dict()) + total_input_tokens += turn.usage.input_tokens + total_output_tokens += turn.usage.output_tokens + total_cache_read_input_tokens += turn.usage.cache_read_input_tokens + total_cache_creation_input_tokens += turn.usage.cache_creation_input_tokens call_indices: dict[str, int] = {} for tool_call in turn.tool_calls: @@ -247,11 +263,11 @@ async def _run_task( # Record the model's calls, but never execute side effects on # a refusal-terminated response. - if stop_reason == "refusal": + if stop_reason is StopReason.REFUSAL: break if not turn.tool_calls: - if stop_reason == "pause_turn" and iterations < max_turns: + if stop_reason is StopReason.PAUSE_TURN and iterations < max_turns: continue break @@ -281,9 +297,12 @@ async def _run_task( backend.add_tool_results(tool_results) if iterations >= max_turns: - hit_max_iterations = stop_reason not in ("end_turn", "max_tokens") + hit_max_iterations = stop_reason not in ( + StopReason.END_TURN, + StopReason.MAX_TOKENS, + ) break - if stop_reason != "tool_use": + if stop_reason is not StopReason.TOOL_USE: break finally: wall_time_s = time.perf_counter() - started_at @@ -313,10 +332,10 @@ async def _run_task( calls[idx]["result_token_count_method"] = TOKEN_ESTIMATE_METHOD if count_estimated else "backend" usage_total = { - "input_tokens": sum(item.get("in", 0) for item in usage_per_iteration), - "output_tokens": sum(item.get("out", 0) for item in usage_per_iteration), - "cache_read_input_tokens": sum(item.get("cache_read", 0) for item in usage_per_iteration), - "cache_creation_input_tokens": sum(item.get("cache_write", 0) for item in usage_per_iteration), + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "cache_read_input_tokens": total_cache_read_input_tokens, + "cache_creation_input_tokens": total_cache_creation_input_tokens, "source": "iterations", } return AgentRun( @@ -325,17 +344,18 @@ async def _run_task( usage=usage_per_iteration[-1] if usage_per_iteration else None, usage_total=usage_total, usage_scope="iteration", - stopped_reason=stop_reason or "end_turn", + stopped_reason=(stop_reason or StopReason.UNKNOWN).value, + provider_stop_reason=provider_stop_reason, call_source="api", hit_max_turns=hit_max_iterations, wall_time_s=round(wall_time_s, 3), usage_per_iteration=usage_per_iteration, - cum_input_tokens=sum(item.get("in", 0) for item in usage_per_iteration), + cum_input_tokens=total_input_tokens, result_pair_mismatch=result_pair_mismatch, token_count_failures=token_count_failures, result_tokens_estimated=result_tokens_estimated, - provider=str(getattr(backend, "provider", self.provider)), - model=str(getattr(backend, "actual_model", getattr(backend, "model", model))), + provider=str(backend.provider), + model=str(backend.actual_model), requested_model=model, ) diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py index ae389fad..a6e54630 100644 --- a/evals/drivers/api/openai.py +++ b/evals/drivers/api/openai.py @@ -5,7 +5,15 @@ import json from typing import Any -from evals.drivers.api.backend import ToolCall, ToolResult, ToolSpec, Turn +from evals.drivers.api.backend import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + Usage, + register_backend, +) def _field(value: Any, name: str, default: Any = None) -> Any: @@ -14,27 +22,29 @@ def _field(value: Any, name: str, default: Any = None) -> Any: return getattr(value, name, default) -def _usage_dict(usage: Any) -> dict[str, int] | None: +def _normalize_usage(usage: Any) -> Usage | None: if usage is None: return None prompt_details = _field(usage, "prompt_tokens_details") - return { - "in": int(_field(usage, "prompt_tokens", 0) or 0), - "out": int(_field(usage, "completion_tokens", 0) or 0), - "cache_read": int(_field(prompt_details, "cached_tokens", 0) or 0), - "cache_write": 0, - } + return Usage( + input_tokens=int(_field(usage, "prompt_tokens", 0) or 0), + output_tokens=int(_field(usage, "completion_tokens", 0) or 0), + cache_read_input_tokens=int(_field(prompt_details, "cached_tokens", 0) or 0), + ) -def _normalize_stop_reason(finish_reason: str | None, refusal: Any) -> str | None: +def _normalize_stop_reason(finish_reason: Any, refusal: Any) -> tuple[StopReason, str | None]: + raw = str(finish_reason) if finish_reason is not None else None if refusal or finish_reason == "content_filter": - return "refusal" - return { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - "function_call": "tool_use", - }.get(str(finish_reason), finish_reason) + return StopReason.REFUSAL, raw + reason = { + "stop": StopReason.END_TURN, + "length": StopReason.MAX_TOKENS, + "tool_calls": StopReason.TOOL_USE, + # Retain support for the predecessor of ``tool_calls``. + "function_call": StopReason.TOOL_USE, + }.get(raw, StopReason.UNKNOWN) + return reason, raw class OpenAIBackend: @@ -138,11 +148,13 @@ def next_turn(self) -> Turn: response_model = _field(completion, "model") if response_model: self.actual_model = str(response_model) + stop_reason, provider_stop_reason = _normalize_stop_reason(_field(choice, "finish_reason"), refusal) return Turn( text=str(content or refusal or ""), tool_calls=calls, - usage=_usage_dict(_field(completion, "usage")), - stop_reason=_normalize_stop_reason(_field(choice, "finish_reason"), refusal), + usage=_normalize_usage(_field(completion, "usage")), + stop_reason=stop_reason, + provider_stop_reason=provider_stop_reason, ) def add_tool_results(self, results: list[ToolResult]) -> None: @@ -156,4 +168,14 @@ def add_tool_results(self, results: list[ToolResult]) -> None: ) +register_backend( + OpenAIBackend.provider, + OpenAIBackend, + model_aliases={ + "standard": "gpt-5.6-sol", + "fast": "gpt-5.6-luna", + }, +) + + __all__ = ["OpenAIBackend"] diff --git a/evals/drivers/base.py b/evals/drivers/base.py index cbbf0644..cdfc51c3 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -56,6 +56,9 @@ class AgentRun: provider: str | None = None model: str | None = None requested_model: str | None = None + # Raw provider finish/stop value. API drivers keep this beside the + # harness-owned normalized ``stopped_reason`` for diagnostics. + provider_stop_reason: str | None = None class AgentDriver(Protocol): @@ -312,6 +315,7 @@ def agent_run_to_harness_dict( "cum_input_tokens_reason": cum_reason, "wall_time_s": run.wall_time_s, "stop_reason": stop_reason, + "provider_stop_reason": run.provider_stop_reason, "hit_max_iterations": hit_max, "result_pair_mismatch": run.result_pair_mismatch, "token_count_failures": run.token_count_failures + local_token_count_failures, diff --git a/evals/run.py b/evals/run.py index c084be28..4d5be019 100644 --- a/evals/run.py +++ b/evals/run.py @@ -12,10 +12,11 @@ from evals import runner from evals.cli import ( - API_MODEL_ALIASES, - CLI_MODEL_ALIASES, + API_MODEL_TIERS, + CLI_DRIVER_PROVIDERS, + CLI_MODEL_TIERS, DEFAULT_OUT_DIR, - MODEL_ALIASES, + MODEL_TIERS, cmd_dry_run, cmd_list, main, @@ -53,15 +54,20 @@ async def run_live( resume: bool = False, record_result_payloads: bool = False, ) -> int: - """Delegate the legacy API while preserving its model-alias behavior.""" - model_id = resolve_model_for_driver(driver_name, model_alias, provider=provider) + """Delegate the legacy API while resolving provider-neutral model tiers.""" + driver_key = (driver_name or "api").strip().lower() + model_id = resolve_model_for_driver( + driver_key, + model_alias, + provider=provider if driver_key in ("api", "sdk") else None, + ) return await runner.run_live( tasks, model_alias=model_alias, reps=reps, surface=surface, out_path=out_path, - driver_name=driver_name, + driver_name=driver_key, provider=provider, server_cmd=server_cmd, server_env=server_env, @@ -72,13 +78,14 @@ async def run_live( __all__ = [ - "API_MODEL_ALIASES", - "CLI_MODEL_ALIASES", + "API_MODEL_TIERS", + "CLI_DRIVER_PROVIDERS", + "CLI_MODEL_TIERS", "DEFAULT_OUT_DIR", "KNOWN_SURFACES", "MAX_ITERATIONS", "MAX_TOKENS", - "MODEL_ALIASES", + "MODEL_TIERS", "classify_call", "cmd_dry_run", "cmd_list", diff --git a/evals/runner.py b/evals/runner.py index 5b147c1e..dfde1a92 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -17,6 +17,7 @@ agent_run_to_harness_dict, get_driver, ) +from evals.drivers.api import MODEL_TIERS from evals.seed import make_plane_client, seed, teardown from evals.tasks import ( PromptBindError, @@ -210,10 +211,13 @@ def load_resume_skip_keys( msg = _resume_field_mismatch(row, field=field, expected=expected) if msg: raise SystemExit(msg) - # New API rows keep both the requested ID (resume identity) and the - # provider-reported model that actually ran. Older rows only have model. + # New tier-aware rows identify the resolved model explicitly. Older + # API rows use requested_model for the resolved ID, while oldest rows + # only have model (which may be provider-reported). model_row = dict(row) - if model_row.get("requested_model"): + if model_row.get("resolved_model"): + model_row["model"] = model_row["resolved_model"] + elif model_row.get("requested_model"): model_row["model"] = model_row["requested_model"] msg = _resume_field_mismatch(model_row, field="model", expected=model) if msg: @@ -245,6 +249,9 @@ def make_run_meta_row( driver: str, git_sha: str, provider: str | None = None, + requested_model: str | None = None, + requested_tier: str | None = None, + resolved_model: str | None = None, ts: str | None = None, ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" @@ -254,6 +261,9 @@ def make_run_meta_row( "surface": surface, "battery": battery, "model": model, + "requested_model": requested_model if requested_model is not None else model, + "requested_tier": requested_tier, + "resolved_model": resolved_model if resolved_model is not None else model, "driver": driver, "provider": provider, "git_sha": git_sha, @@ -318,6 +328,8 @@ def _base_row( driver_name: str, provider: str | None, model_id: str | None, + model_request: str | None, + requested_tier: str | None, task: dict[str, Any], rep: int, battery: str, @@ -333,7 +345,9 @@ def _base_row( "provider": provider, "classification": classification, "model": model_id, - "requested_model": model_id, + "requested_model": model_request, + "requested_tier": requested_tier, + "resolved_model": model_id, "task_id": task["id"], "author": task_author(task), "rep": rep, @@ -344,6 +358,7 @@ def _base_row( "error_class": None, "final_text": "", "stop_reason": None, + "provider_stop_reason": None, "hit_max_iterations": False, "result_pair_mismatch": False, "token_count_failures": 0, @@ -396,6 +411,7 @@ async def run_live( is_api_driver = driver_name in ("api", "sdk") provider_id = provider if is_api_driver else None model_id = resolved_model_id if resolved_model_id is not None else model_alias + requested_tier = model_alias if model_alias in MODEL_TIERS else None run_id = uuid.uuid4().hex git_sha = _git_sha() @@ -424,6 +440,9 @@ async def run_live( surface=surface, battery=battery, model=model_id, + requested_model=model_alias, + requested_tier=requested_tier, + resolved_model=model_id, driver=driver_name, provider=provider_id, git_sha=git_sha, @@ -447,7 +466,8 @@ async def run_live( driver = get_driver(driver_name, **driver_kwargs) print( - f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} model={model_id} " + f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} " + f"requested_model={model_alias} resolved_model={model_id} " f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" ) print(f"writing {out_path}") @@ -479,6 +499,8 @@ async def _run_tasks() -> None: driver_name=driver_name, provider=provider_id, model_id=model_id, + model_request=model_alias, + requested_tier=requested_tier, task=task, rep=rep, battery=battery, @@ -566,6 +588,11 @@ async def _run_tasks() -> None: if agent is not None: row.update(agent) + # Driver-level requested_model is the resolved ID. + # Restore run-level intent and retain both identities. + row["requested_model"] = model_alias + row["requested_tier"] = requested_tier + row["resolved_model"] = model_id if external: # Empty overlay sets would classify every call # out-of-set; null the counters instead. diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py index 93790475..a7194c70 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/test_evals_api_driver.py @@ -8,15 +8,24 @@ from types import SimpleNamespace from typing import Any +import pytest + from evals.drivers import agent_run_to_harness_dict from evals.drivers.api import ( + KNOWN_API_PROVIDERS, AnthropicBackend, ApiDriver, OpenAIBackend, + StopReason, ToolCall, ToolResult, ToolSpec, Turn, + UnmappedModelTierError, + Usage, + register_backend, + resolve_backend_model, + unregister_backend, ) from evals.token_counting import estimate_result_tokens @@ -99,26 +108,81 @@ def run_driver(driver: ApiDriver, *, max_turns: int = 5): ) +def test_registered_third_party_backend_runs_without_driver_changes(): + created: list[FakeBackend] = [] + + class DummyBackend(FakeBackend): + provider = "dummy" + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + super().__init__( + [ + Turn( + text=f"done in {max_tokens}", + tool_calls=[], + usage=Usage(input_tokens=7, output_tokens=2), + stop_reason=StopReason.END_TURN, + provider_stop_reason="dummy_complete", + ) + ] + ) + self.model = model + self.actual_model = f"{model}-actual" + self.client = client + created.append(self) + + session = FakeMcpSession() + + @asynccontextmanager + async def session_factory(_params): + yield session + + register_backend("dummy", DummyBackend) + try: + assert "dummy" in KNOWN_API_PROVIDERS + with pytest.raises(UnmappedModelTierError, match=r"standard.*explicit model ID"): + resolve_backend_model("dummy", "standard") + assert resolve_backend_model("dummy", "dummy-explicit") == "dummy-explicit" + driver = ApiDriver( + provider="dummy", + client=object(), + mcp_session_factory=session_factory, + max_tokens=99, + ) + run = driver.run_task("do it", {"SAFE": "1"}, "dummy-model", 1) + finally: + unregister_backend("dummy") + + assert created[0].started is not None + assert run.final_text == "done in 99" + assert run.provider == "dummy" + assert run.model == "dummy-model-actual" + assert run.stopped_reason == "end_turn" + assert run.provider_stop_reason == "dummy_complete" + assert run.usage_per_iteration == [{"in": 7, "out": 2, "cache_read": 0, "cache_write": 0}] + + def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): backend = FakeBackend( [ Turn( text="", tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], - usage={"in": 10, "out": 2, "cache_read": 3, "cache_write": 1}, - stop_reason="tool_use", + usage=Usage(10, 2, 3, 1), + stop_reason=StopReason.TOOL_USE, ), Turn( text="", tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], - usage={"in": 20, "out": 4, "cache_read": 6, "cache_write": 0}, - stop_reason="tool_use", + usage=Usage(20, 4, 6, 0), + stop_reason=StopReason.TOOL_USE, ), Turn( text="done", tool_calls=[], - usage={"in": 30, "out": 6, "cache_read": 9, "cache_write": 0}, - stop_reason="end_turn", + usage=Usage(30, 6, 9, 0), + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", ), ] ) @@ -154,6 +218,7 @@ def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): assert run.token_count_failures == 0 assert run.provider == "fake" assert run.model == "fake-actual" + assert run.provider_stop_reason == "fake_done" def test_api_driver_refusal_records_calls_but_executes_nothing(): @@ -162,8 +227,8 @@ def test_api_driver_refusal_records_calls_but_executes_nothing(): Turn( text="declined", tool_calls=[ToolCall("write-1", "write", {"value": "x"})], - usage={"in": 1, "out": 1, "cache_read": 0, "cache_write": 0}, - stop_reason="refusal", + usage=Usage(1, 1), + stop_reason=StopReason.REFUSAL, ) ] ) @@ -185,9 +250,9 @@ def test_api_driver_pairs_results_by_id_not_ordinal(): text="", tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], usage=None, - stop_reason="tool_use", + stop_reason=StopReason.TOOL_USE, ), - Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), ] ) # The fake session deliberately returns tagged results in reverse ID order. @@ -211,9 +276,14 @@ def test_api_driver_flags_result_id_mismatch(): text="", tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], usage=None, - stop_reason="tool_use", + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, ), - Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), ] ) session = FakeMcpSession( @@ -236,9 +306,14 @@ def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): text="", tool_calls=[ToolCall("a", "lookup", {"q": "a"})], usage=None, - stop_reason="tool_use", + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="must not be read", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, ), - Turn(text="must not be read", tool_calls=[], usage=None, stop_reason="end_turn"), ] ) session = FakeMcpSession([ToolResult(call_id="a", text="result")]) @@ -253,7 +328,7 @@ def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): def test_api_driver_clean_end_on_last_iteration_is_not_capped(): - backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn")]) + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) @@ -268,9 +343,9 @@ def test_api_driver_uses_optional_backend_token_counter(): text="", tool_calls=[ToolCall("a", "lookup", {"q": "a"})], usage=None, - stop_reason="tool_use", + stop_reason=StopReason.TOOL_USE, ), - Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), ] ) backend.count_tokens = lambda text: len(text) + 10 @@ -288,10 +363,16 @@ def test_api_driver_maps_every_legacy_row_field(): Turn( text="", tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage={"in": 4, "out": 1, "cache_read": 0, "cache_write": 0}, - stop_reason="tool_use", + usage=Usage(4, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", ), - Turn(text="done", tool_calls=[], usage=None, stop_reason="end_turn"), ] ) run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) @@ -316,6 +397,7 @@ def test_api_driver_maps_every_legacy_row_field(): "cum_input_tokens", "wall_time_s", "stop_reason", + "provider_stop_reason", "hit_max_iterations", "result_pair_mismatch", "token_count_failures", @@ -336,6 +418,7 @@ def test_api_driver_maps_every_legacy_row_field(): assert row["provider"] == "fake" assert row["model"] == "fake-actual" assert row["requested_model"] == "fake-requested" + assert row["provider_stop_reason"] == "fake_done" class FakeAnthropicMessages: @@ -348,6 +431,29 @@ def create(self, **kwargs): return self.responses.popleft() +@pytest.mark.parametrize( + ("raw_reason", "expected"), + [ + ("end_turn", StopReason.END_TURN), + ("tool_use", StopReason.TOOL_USE), + ("max_tokens", StopReason.MAX_TOKENS), + ("refusal", StopReason.REFUSAL), + ("pause_turn", StopReason.PAUSE_TURN), + ("model_context_window_exceeded", StopReason.MODEL_CONTEXT_WINDOW_EXCEEDED), + ("future_reason", StopReason.UNKNOWN), + ], +) +def test_anthropic_backend_normalizes_and_preserves_stop_reason(raw_reason, expected): + messages = FakeAnthropicMessages([{"model": "claude", "content": [], "usage": None, "stop_reason": raw_reason}]) + backend = AnthropicBackend("claude", max_tokens=10, client=SimpleNamespace(messages=messages)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason is expected + assert turn.provider_stop_reason == raw_reason + + def test_anthropic_backend_translates_tools_turns_and_results(): responses = [ { @@ -390,7 +496,9 @@ def test_anthropic_backend_translates_tools_turns_and_results(): {"name": "lookup", "description": "Look up", "input_schema": {"type": "object", "required": ["q"]}} ] assert first.tool_calls == [ToolCall("toolu-1", "lookup", {"q": "x"})] - assert first.usage == {"in": 10, "out": 2, "cache_read": 3, "cache_write": 4} + assert first.usage == Usage(10, 2, 3, 4) + assert first.stop_reason is StopReason.TOOL_USE + assert first.provider_stop_reason == "tool_use" replay = messages.requests[1]["messages"] assert replay[1] == {"role": "assistant", "content": responses[0]["content"]} assert replay[2] == { @@ -418,6 +526,44 @@ def create(self, **kwargs): return self.responses.popleft() +@pytest.mark.parametrize( + ("raw_reason", "expected"), + [ + ("stop", StopReason.END_TURN), + ("tool_calls", StopReason.TOOL_USE), + ("length", StopReason.MAX_TOKENS), + ("content_filter", StopReason.REFUSAL), + ("future_reason", StopReason.UNKNOWN), + ], +) +def test_openai_backend_normalizes_and_preserves_stop_reason(raw_reason, expected): + completions = FakeOpenAICompletions( + [ + { + "model": "gpt", + "choices": [ + { + "finish_reason": raw_reason, + "message": {"content": "done", "tool_calls": []}, + } + ], + "usage": None, + } + ] + ) + backend = OpenAIBackend( + "gpt", + max_tokens=10, + client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), + ) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason is expected + assert turn.provider_stop_reason == raw_reason + + def test_openai_backend_translates_tools_calls_and_tool_messages(): responses = [ { @@ -476,8 +622,9 @@ def test_openai_backend_translates_tools_calls_and_tool_messages(): } ] assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] - assert first.stop_reason == "tool_use" - assert first.usage == {"in": 12, "out": 3, "cache_read": 5, "cache_write": 0} + assert first.stop_reason is StopReason.TOOL_USE + assert first.provider_stop_reason == "tool_calls" + assert first.usage == Usage(12, 3, 5, 0) second_messages = completions.requests[1]["messages"] assert second_messages[2] == { "role": "assistant", @@ -492,7 +639,8 @@ def test_openai_backend_translates_tools_calls_and_tool_messages(): } assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} assert second.text == "done" - assert second.stop_reason == "end_turn" + assert second.stop_reason is StopReason.END_TURN + assert second.provider_stop_reason == "stop" assert backend.actual_model == "gpt-actual" @@ -530,6 +678,7 @@ def test_openai_backend_normalizes_refusal_for_driver_guard(): turn = backend.next_turn() - assert turn.stop_reason == "refusal" + assert turn.stop_reason is StopReason.REFUSAL + assert turn.provider_stop_reason == "content_filter" assert turn.text == "declined" assert turn.tool_calls == [ToolCall("danger", "write", {})] diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index 8d11067e..db66672e 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -861,6 +861,7 @@ def test_parse_args_accepts_driver(): assert a.driver == "claude-cli" b = parse_args(["--dry-run"]) assert b.driver == "api" + assert b.model == "standard" assert b.provider == "anthropic" assert b.record_result_payloads is False c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 90976470..215a5f17 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -147,6 +147,31 @@ def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): assert ("R1", 0) in skip +def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): + p = tmp_path / "tiered.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "surface": "v2", + "model": "provider-reported-id", + "requested_model": "standard", + "requested_tier": "standard", + "resolved_model": "old-standard-id", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + + skip, _, _ = load_resume_skip_keys(p, surface="v2", model="old-standard-id") + assert skip == {("R1", 0)} + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, surface="v2", model="new-standard-id") + + def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): p = tmp_path / "out.jsonl" p.write_text( @@ -206,7 +231,7 @@ def boom_seed(plane, run_id, needs, ctx): rc = asyncio.run( run_live( [task], - model_alias="sonnet", + model_alias="standard", reps=1, surface="full", out_path=out, @@ -222,6 +247,13 @@ def boom_seed(plane, run_id, needs, ctx): assert "HttpError" in (row["error"] or "") assert "identifier" in (row["error"] or "").lower() assert row["battery"] # fingerprint written + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "sonnet" + assert row["model"] == "sonnet" + meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert meta["requested_tier"] == "standard" + assert meta["resolved_model"] == "sonnet" def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index aaf89426..daa7e3b8 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -39,6 +39,7 @@ write_all_fd, ) from evals.proxy import main as proxy_main +from evals.run import main as eval_main from evals.run import resolve_model_for_driver from evals.token_counting import estimate_result_tokens @@ -1041,15 +1042,68 @@ def fake_run(cmd, **kwargs): assert run.call_source != "proxy" -def test_resolve_model_for_driver_qualification(): - assert resolve_model_for_driver("claude-cli", "sonnet") == "sonnet" - assert resolve_model_for_driver("opencode-cli", "sonnet").startswith("anthropic/") - assert resolve_model_for_driver("antigravity-cli", "haiku").startswith("gemini") - # Free-form passthrough +def test_model_tiers_resolve_per_driver_and_provider(): + assert resolve_model_for_driver("api", "standard", provider="anthropic") == "claude-sonnet-5" + assert resolve_model_for_driver("api", "fast", provider="anthropic") == "claude-haiku-4-5" + assert resolve_model_for_driver("api", "standard", provider="openai") == "gpt-5.6-sol" + assert resolve_model_for_driver("api", "fast", provider="openai") == "gpt-5.6-luna" + assert resolve_model_for_driver("sdk", "standard", provider="openai") == "gpt-5.6-sol" + assert resolve_model_for_driver("claude-cli", "standard") == "sonnet" + assert resolve_model_for_driver("claude-cli", "fast") == "haiku" + assert resolve_model_for_driver("codex-cli", "standard") == "gpt-5.6-sol" + assert resolve_model_for_driver("codex-cli", "fast") == "gpt-5.6-luna" + assert resolve_model_for_driver("antigravity-cli", "standard") == "gemini-3.6-flash-high" + assert resolve_model_for_driver("antigravity-cli", "fast") == "gemini-3.6-flash-low" + + +@pytest.mark.parametrize( + ("driver", "model"), + [ + ("api", "claude-opus-5"), + ("api", "sonnet"), + ("claude-cli", "sonnet"), + ("codex-cli", "sonnet"), + ("antigravity-cli", "gemini-3.1-pro-high"), + ("opencode-cli", "haiku"), + ], +) +def test_non_tier_model_strings_pass_through_unchanged(driver, model): + assert resolve_model_for_driver(driver, model) == model + + +def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): + with pytest.raises(ValueError, match=r"opencode models"): + resolve_model_for_driver("opencode-cli", "standard") + + +def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path: Path, capsys): + out = tmp_path / "must-not-exist.jsonl" + + rc = eval_main( + [ + "--driver", + "opencode-cli", + "--model", + "standard", + "--tasks", + "R1", + "--out", + str(out), + ] + ) + + assert rc == 2 + assert "explicit provider/model ID" in capsys.readouterr().err + assert out.exists() is False + + +def test_tier_mapping_is_scoped_to_cli_provider(): + with pytest.raises(ValueError, match=r"codex-cli.*anthropic.*explicit model ID"): + resolve_model_for_driver("codex-cli", "standard", provider="anthropic") + + +def test_qualified_model_id_passes_through_unchanged(): assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" - assert resolve_model_for_driver("sdk", "sonnet") == "claude-sonnet-5" - assert resolve_model_for_driver("api", "haiku") == "claude-haiku-4-5" - assert resolve_model_for_driver("api", "sonnet", provider="openai") == "gpt-5" def test_ensure_proxy_pythonpath_injects_repo(): From b8f46171886ffb5e3e2b55795a73524b910df4d4 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 10:10:50 +0530 Subject: [PATCH 09/93] Collapse the four CLI driver bodies into one template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every CLI driver repeated the same sequence: temp dir, write the vendor MCP config, wrap the server command in the recording proxy, run it, parse the vendor's output, harvest the proxy on timeout, reconcile, assemble an AgentRun. Four copies of one algorithm — and the duplication had already cost us, because the process-group kill, the timeout harvest and a wrong-server bug each had to be fixed in more than one copy, with no way to tell whether a copy was missed. CliDriver now owns that sequence once and the vendors express only what differs: write_mcp_config, build_command, invoke_cli, parse_output, plus validate_run and finalize_run for the two drivers that need them. Claude, Codex, Antigravity and OpenCode drop 250 lines between them. Free functions stayed free on purpose. strip_mcp_prefix, normalize_tool_call, normalize_claude_usage, the vendor parsers and the token estimator have no vendor polymorphism and no state; as methods they would need an object to test and would drift toward coupling to self. The line that matters is whether shared logic dispatches on it, not whether it happens to live near a class. Co-Authored-By: Claude Fable 5 --- evals/drivers/__init__.py | 2 + evals/drivers/antigravity.py | 236 ++++++++++----------------- evals/drivers/claude.py | 308 ++++++++++++++--------------------- evals/drivers/cli.py | 280 +++++++++++++++++++++++++++++++ evals/drivers/codex.py | 296 ++++++++++++++------------------- evals/drivers/opencode.py | 224 +++++++++---------------- 6 files changed, 689 insertions(+), 657 deletions(-) create mode 100644 evals/drivers/cli.py diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 8b338755..49ada485 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -62,6 +62,7 @@ parse_claude_transcript_calls, write_claude_mcp_config, ) +from evals.drivers.cli import CliDriver from evals.drivers.codex import ( CodexCliDriver, find_codex_rollout, @@ -110,6 +111,7 @@ def get_driver(name: str, **kwargs: Any) -> AgentDriver: "AntigravityCliDriver", "ApiDriver", "ClaudeCliDriver", + "CliDriver", "CodexCliDriver", "OpencodeCliDriver", "REPO_ROOT", diff --git a/evals/drivers/antigravity.py b/evals/drivers/antigravity.py index d7758ca8..8043d544 100644 --- a/evals/drivers/antigravity.py +++ b/evals/drivers/antigravity.py @@ -5,21 +5,10 @@ import json import os import subprocess -import sys -import tempfile -import time from collections.abc import Callable from pathlib import Path -from typing import Any -from evals.drivers.base import REPO_ROOT, AgentRun -from evals.drivers.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - proxy_wrap_server_command, -) +from evals.drivers.cli import CliDriver, CliLaunch, CliOutput # Antigravity CLI (agy) — proxy-first # --------------------------------------------------------------------------- @@ -105,7 +94,7 @@ def prepare_antigravity_fake_home( ) -class AntigravityCliDriver: +class AntigravityCliDriver(CliDriver): """Run tasks via Google Antigravity CLI (``agy``). Probed flags (2026-08-12, ``agy --help``): @@ -121,6 +110,10 @@ class AntigravityCliDriver: """ name = "antigravity-cli" + run_notes = ("no_turn_cap",) + temp_dir_prefix = "plane-eval-antigravity-" + exit_note_prefix = "agy" + include_stderr_in_exit_note = True def __init__( self, @@ -133,147 +126,94 @@ def __init__( record_result_payloads: bool = False, ) -> None: self.agy_bin = agy_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + fake_home = temp_dir / "home" + prepare_antigravity_fake_home( + fake_home, + command=server_command[0], + args=server_command[1:], + env=child_env, + ) + run_env = {**os.environ, "HOME": str(fake_home)} + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + return CliLaunch(cwd=task_cwd, env=run_env) - def run_task( + def build_command( self, prompt: str, - mcp_env: dict[str, str], + *, model: str | None, max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns, launch + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + command = [ + self.agy_bin, + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + ] + if model: + command.extend(["--model", model]) + command.append(full_prompt) + return command + + def invoke_cli( + self, + command: list[str], *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = ["no_turn_cap"] - t0 = time.perf_counter() - child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - - with tempfile.TemporaryDirectory(prefix="plane-eval-antigravity-") as td: - td_path = Path(td) - sidecar = td_path / "proxy-sidecar.jsonl" - fake_home = td_path / "home" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command( - real_cmd, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - ) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env_plane = ensure_proxy_pythonpath(child_env_plane) - else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - prepare_antigravity_fake_home( - fake_home, - command=server_cmd, - args=server_args, - env=child_env_plane, - ) - - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd: list[str] = [ - self.agy_bin, - "-p", - "--output-format", - "json", - "--dangerously-skip-permissions", - ] - if model: - cmd.extend(["--model", model]) - cmd.append(full_prompt) - - run_env = {**os.environ, "HOME": str(fake_home)} - if "PATH" in child_env_plane: - run_env["PATH"] = child_env_plane["PATH"] - - timeout_s = max(120, max_turns * 60) - try: - try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - env=run_env, - ) - except TypeError: - # Some test runners reject ``env=``; retry without it. - # TimeoutExpired from this path must still hit the harvest below. - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - usage_scope="run", - call_source=call_source, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proc.returncode != 0: - notes.append(f"agy_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - - final_text = stdout.strip() - try: - if final_text.lstrip().startswith("{"): - blob = json.loads(final_text) - if isinstance(blob, dict): - final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) - except json.JSONDecodeError: - pass - - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=final_text, - usage=None, - stopped_reason="error" if proc.returncode else "end_turn", - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) + launch: CliLaunch, + timeout_s: int, + ) -> subprocess.CompletedProcess[str]: + try: + return super().invoke_cli(command, launch=launch, timeout_s=timeout_s) + except TypeError: + # Some test runners reject ``env=``; retry without it. A timeout + # from this fallback still reaches the template's harvest path. + fallback = CliLaunch(cwd=launch.cwd, config_args=launch.config_args) + return super().invoke_cli(command, launch=fallback, timeout_s=timeout_s) + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del task_cwd, max_turns + final_text = (proc.stdout or "").strip() + try: + if final_text.lstrip().startswith("{"): + blob = json.loads(final_text) + if isinstance(blob, dict): + final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) + except json.JSONDecodeError: + pass + + return CliOutput( + final_text=final_text, + stopped_reason="error" if proc.returncode else "end_turn", + ) __all__ = [ diff --git a/evals/drivers/claude.py b/evals/drivers/claude.py index cb810f6a..82d566fd 100644 --- a/evals/drivers/claude.py +++ b/evals/drivers/claude.py @@ -4,26 +4,12 @@ import json import subprocess -import sys -import tempfile -import time from collections.abc import Callable from pathlib import Path from typing import Any -from evals.drivers.base import ( - REPO_ROOT, - AgentRun, - normalize_tool_call, - split_plane_and_client_calls, -) -from evals.drivers.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - proxy_wrap_server_command, -) +from evals.drivers.base import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.cli import CliDriver, CliLaunch, CliOutput, CliOutputError def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: @@ -284,10 +270,11 @@ def write_claude_mcp_config( # --------------------------------------------------------------------------- -class ClaudeCliDriver: +class ClaudeCliDriver(CliDriver): """Run tasks via ``claude -p`` on the user's Claude Code subscription.""" name = "claude-cli" + temp_dir_prefix = "plane-eval-claude-" def __init__( self, @@ -302,194 +289,135 @@ def __init__( record_result_payloads: bool = False, ) -> None: self.claude_bin = claude_bin - self.python_bin = python_bin or sys.executable self.permission_mode = permission_mode self.strict_mcp = strict_mcp - self._runner = runner or run_cli_subprocess # Full replacement for the MCP server launch (external surfaces under # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads - - def run_task( + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + mcp_cfg = temp_dir / "mcp.json" + write_claude_mcp_config( + mcp_cfg, + command=server_command[0], + args=server_command[1:], + env=child_env, + server_name="plane", + ) + return CliLaunch(cwd=task_cwd, config_args=["--mcp-config", str(mcp_cfg)]) + + def build_command( self, prompt: str, - mcp_env: dict[str, str], + *, model: str | None, max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + command = [ + self.claude_bin, + "-p", + "--output-format", + "json", + *launch.config_args, + "--permission-mode", + self.permission_mode, + "--max-turns", + str(max_turns), + ] + if self.strict_mcp: + command.append("--strict-mcp-config") + if model: + command.extend(["--model", model]) + if system: + command.extend(["--append-system-prompt", system]) + # --allowedTools is variadic and would swallow the trailing prompt. + command.extend(["--allowedTools=mcp__plane__*", prompt]) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = [] - t0 = time.perf_counter() - - with tempfile.TemporaryDirectory(prefix="plane-eval-claude-") as td: - td_path = Path(td) - mcp_cfg = td_path / "mcp.json" - sidecar = td_path / "proxy-sidecar.jsonl" - # Only pass Plane-related env into the MCP child (plus PATH/HOME if present). - child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command( - real_cmd, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - ) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env = ensure_proxy_pythonpath(child_env) - else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - write_claude_mcp_config( - mcp_cfg, - command=server_cmd, - args=server_args, - env=child_env, - server_name="plane", - ) - - cmd: list[str] = [ - self.claude_bin, - "-p", - "--output-format", - "json", - "--mcp-config", - str(mcp_cfg), - "--permission-mode", - self.permission_mode, - "--max-turns", - str(max_turns), - ] - if self.strict_mcp: - cmd.append("--strict-mcp-config") - if model: - cmd.extend(["--model", model]) - if system: - cmd.extend(["--append-system-prompt", system]) - # Allow MCP tools from our server without interactive prompts - # --allowedTools is variadic and would swallow the trailing prompt; use = form. - cmd.append("--allowedTools=mcp__plane__*") - cmd.append(prompt) - - timeout_s = max(120, max_turns * 60) + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + stdout = proc.stdout or "" + stderr = proc.stderr or "" + parsed: dict[str, Any] | None = None + parse_err: str | None = None + # JSON may be the whole stdout or the last JSON object line. + for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): + if not candidate or not candidate.lstrip().startswith("{"): + continue try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - # Wait for proxy finalization before harvesting / temp dir teardown. - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = harvest_proxy_after_cli_timeout( - calls, client_calls, sidecar, notes - ) - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - - parsed: dict[str, Any] | None = None - parse_err: str | None = None - # JSON may be the whole stdout or the last JSON object line - for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): - if not candidate or not candidate.lstrip().startswith("{"): - continue - try: - parsed = parse_claude_json_result(candidate) - break - except (json.JSONDecodeError, ValueError, TypeError) as exc: - parse_err = str(exc) - continue + parsed = parse_claude_json_result(candidate) + break + except (json.JSONDecodeError, ValueError, TypeError) as exc: + parse_err = str(exc) - if parsed is None: - notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") - if proc.returncode != 0: - notes.append(f"claude_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - if self.use_proxy: - apply_proxy_sidecar([], [], sidecar, notes) - detail = "; ".join(notes) - raise RuntimeError(f"claude cli failed: {detail}") - - # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). + if parsed is None: + notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") if proc.returncode != 0: notes.append(f"claude_exit={proc.returncode}") if stderr.strip(): notes.append(stderr.strip()[:500]) - - # JSON rarely embeds per-call tool detail — prefer transcript when present. - calls = list(parsed.get("calls") or []) - client_calls = list(parsed.get("client_tool_calls") or []) - call_source = "json" if (calls or client_calls) else "json" - session_id = parsed.get("session_id") - transcript = find_claude_transcript(session_id, cwd) - if transcript is not None: - tagged = parse_claude_transcript_calls(transcript) - t_plane, t_client = split_plane_and_client_calls(tagged) - if t_plane or t_client: - calls, client_calls = t_plane, t_client - call_source = "transcript" - notes.append(f"calls_from_transcript:{transcript}") - if not calls and not client_calls: - notes.append("no_tool_calls_in_json_or_transcript") - - # Proxy sidecar (when enabled) replaces CLI-parsed plane calls. - if self.use_proxy: - calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proxy_src == "proxy": - call_source = "proxy" - - num_turns = parsed.get("num_turns") - hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) - stopped = parsed["stopped_reason"] - if hit_max and stopped in ("end_turn", "completed", ""): - stopped = "max_turns" - - raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=parsed["final_text"], - usage=parsed.get("usage"), - usage_total=parsed.get("usage_total"), - stopped_reason=stopped, - raw_ref=raw_ref, - usage_scope="run", - call_source=call_source, - hit_max_turns=hit_max, - wall_time_s=round(wall, 3), - notes=notes, - ) + raise CliOutputError("claude cli failed") + + # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "json" + session_id = parsed.get("session_id") + transcript = find_claude_transcript(session_id, task_cwd) + if transcript is not None: + tagged = parse_claude_transcript_calls(transcript) + transcript_plane, transcript_client = split_plane_and_client_calls(tagged) + if transcript_plane or transcript_client: + calls, client_calls = transcript_plane, transcript_client + call_source = "transcript" + notes.append(f"calls_from_transcript:{transcript}") + if not calls and not client_calls: + notes.append("no_tool_calls_in_json_or_transcript") + + num_turns = parsed.get("num_turns") + hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) + stopped = parsed["stopped_reason"] + if hit_max and stopped in ("end_turn", "completed", ""): + stopped = "max_turns" + + raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) + return CliOutput( + calls=calls, + final_text=parsed["final_text"], + client_tool_calls=client_calls, + usage=parsed.get("usage"), + usage_total=parsed.get("usage_total"), + stopped_reason=stopped, + raw_ref=raw_ref, + call_source=call_source, + hit_max_turns=hit_max, + ) __all__ = [ diff --git a/evals/drivers/cli.py b/evals/drivers/cli.py new file mode 100644 index 00000000..960a8157 --- /dev/null +++ b/evals/drivers/cli.py @@ -0,0 +1,280 @@ +"""Shared template for subprocess-backed CLI eval drivers.""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import time +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from evals.drivers.base import REPO_ROOT, AgentRun +from evals.drivers.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) + + +@dataclass +class CliLaunch: + """Vendor-prepared CLI launch details.""" + + cwd: Path + config_args: list[str] = field(default_factory=list) + env: dict[str, str] | None = None + + +@dataclass +class CliOutput: + """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" + + final_text: str + calls: list[dict[str, Any]] = field(default_factory=list) + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + usage: dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + stopped_reason: str = "end_turn" + raw_ref: str | None = None + call_source: str = "json" + hit_max_turns: bool = False + + +class CliOutputError(RuntimeError): + """Signal that vendor output could not produce a valid ``AgentRun``.""" + + +class CliDriver(ABC): + """Template for CLI drivers that run one MCP-backed subprocess task.""" + + name: str + experimental = False + default_call_source = "json" + run_notes: tuple[str, ...] = () + temp_dir_prefix = "plane-eval-cli-" + temp_dir_in_cwd = False + include_setup_in_wall_time = True + exit_note_prefix: str | None = None + include_stderr_in_exit_note = False + + def __init__( + self, + *, + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads + + def validate_run(self) -> None: + """Reject a launch before any temporary state is created, if needed.""" + return None + + @abstractmethod + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + """Write vendor MCP configuration and return launch settings.""" + + @abstractmethod + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + """Build the vendor CLI command.""" + + def invoke_cli( + self, + command: list[str], + *, + launch: CliLaunch, + timeout_s: int, + ) -> subprocess.CompletedProcess[str]: + """Invoke the configured runner with the shared subprocess contract.""" + kwargs: dict[str, Any] = { + "cwd": str(launch.cwd), + "capture_output": True, + "text": True, + "timeout": timeout_s, + } + if launch.env is not None: + kwargs["env"] = launch.env + return self._runner(command, **kwargs) + + @abstractmethod + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + """Parse vendor output into the normalized CLI result shape.""" + + def finalize_run( + self, + proc: subprocess.CompletedProcess[str], + *, + output: CliOutput, + notes: list[str], + ) -> None: + """Apply vendor handling that must occur after proxy reconciliation.""" + del output + if proc.returncode != 0 and self.exit_note_prefix: + notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") + stderr = proc.stderr or "" + if self.include_stderr_in_exit_note and stderr.strip(): + notes.append(stderr.strip()[:500]) + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + """Run one CLI task using the shared configuration/proxy/timeout flow.""" + task_cwd = (cwd or REPO_ROOT).resolve() + notes = list(self.run_notes) + self.validate_run() + started_at = time.perf_counter() if self.include_setup_in_wall_time else None + temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None + + with tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td: + temp_dir = Path(td) + sidecar = temp_dir / "proxy-sidecar.jsonl" + child_env = { + key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") + } + real_command = ( + list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] + ) + server_command = real_command + if self.use_proxy: + server_command = proxy_wrap_server_command( + real_command, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) + child_env = ensure_proxy_pythonpath(child_env) + + launch = self.write_mcp_config( + temp_dir, + task_cwd=task_cwd, + server_command=server_command, + child_env=child_env, + ) + command = self.build_command( + prompt, + model=model, + max_turns=max_turns, + system=system, + launch=launch, + ) + if started_at is None: + started_at = time.perf_counter() + timeout_s = max(120, max_turns * 60) + + try: + proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - started_at + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = self.default_call_source + if self.use_proxy: + calls, client_calls, call_source = harvest_proxy_after_cli_timeout( + calls, + client_calls, + sidecar, + notes, + ) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + experimental=self.experimental, + notes=notes, + ) + + wall = time.perf_counter() - started_at + try: + output = self.parse_output( + proc, + task_cwd=task_cwd, + max_turns=max_turns, + notes=notes, + ) + except CliOutputError as exc: + if self.use_proxy: + apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise RuntimeError(f"{exc}: {detail}") from None + + if self.use_proxy: + calls, client_calls, proxy_source = apply_proxy_sidecar( + output.calls, + output.client_tool_calls, + sidecar, + notes, + ) + output.calls = calls + output.client_tool_calls = client_calls + if proxy_source == "proxy": + output.call_source = "proxy" + + self.finalize_run(proc, output=output, notes=notes) + return AgentRun( + calls=output.calls, + client_tool_calls=output.client_tool_calls, + final_text=output.final_text, + usage=output.usage, + usage_total=output.usage_total, + stopped_reason=output.stopped_reason, + raw_ref=output.raw_ref, + usage_scope="run", + call_source=output.call_source, + hit_max_turns=output.hit_max_turns, + wall_time_s=round(wall, 3), + experimental=self.experimental, + notes=notes, + ) + + +__all__ = ["CliDriver", "CliLaunch", "CliOutput", "CliOutputError"] diff --git a/evals/drivers/codex.py b/evals/drivers/codex.py index a179ce00..a29be44f 100644 --- a/evals/drivers/codex.py +++ b/evals/drivers/codex.py @@ -4,26 +4,13 @@ import json import subprocess -import sys -import tempfile -import time from collections.abc import Callable from pathlib import Path from typing import Any -from evals.drivers.base import ( - REPO_ROOT, - AgentRun, - normalize_tool_call, - split_plane_and_client_calls, -) -from evals.drivers.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - proxy_wrap_server_command, -) +from evals.drivers.base import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.cli import CliDriver, CliLaunch, CliOutput +from evals.drivers.process import run_cli_subprocess def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: @@ -237,7 +224,7 @@ def write_codex_mcp_override_args( # --------------------------------------------------------------------------- -class CodexCliDriver: +class CodexCliDriver(CliDriver): """Run tasks via ``codex exec`` (experimental; metered quota). Live invocation is supported for the interface, but the eval harness should @@ -247,6 +234,11 @@ class CodexCliDriver: name = "codex-cli" experimental = True + default_call_source = "stream" + run_notes = ("experimental:codex-cli",) + temp_dir_prefix = "plane-eval-codex-" + include_setup_in_wall_time = False + exit_note_prefix = "codex" def __init__( self, @@ -260,173 +252,127 @@ def __init__( record_result_payloads: bool = False, ) -> None: self.codex_bin = codex_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess self.allow_live = allow_live - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - cwd = (cwd or REPO_ROOT).resolve() - notes = ["experimental:codex-cli"] + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def validate_run(self) -> None: if self._runner is run_cli_subprocess and not self.allow_live: raise RuntimeError( "CodexCliDriver refuses live runs by default (metered weekly quota). " "Pass allow_live=True or inject a fake runner for tests." ) - child_env = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - with tempfile.TemporaryDirectory(prefix="plane-eval-codex-") as td: - td_path = Path(td) - sidecar = td_path / "proxy-sidecar.jsonl" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - wrapped = proxy_wrap_server_command( - real_cmd, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - ) - server_cmd, server_args = wrapped[0], wrapped[1:] - child_env = ensure_proxy_pythonpath(child_env) + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir + config_args = write_codex_mcp_override_args( + command=server_command[0], + args=server_command[1:], + env=child_env, + server_name="plane", + ) + return CliLaunch(cwd=task_cwd, config_args=config_args) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns + command = [ + self.codex_bin, + "exec", + "--json", + "--skip-git-repo-check", + *launch.config_args, + ] + if model: + command.extend(["-m", model]) + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + command.append(full_prompt) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del task_cwd, max_turns + parsed = parse_codex_jsonl_events(proc.stdout or "") + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "stream" + session_id = parsed.get("session_id") + + # Exact-match rollout only — never steal another parallel task's file. + need_rollout = (not calls and not client_calls) or not parsed.get("final_text") + if need_rollout and session_id: + rollout = find_codex_rollout(session_id) + if rollout is not None: + full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) + if not calls and not client_calls: + calls = list(full.get("calls") or []) + client_calls = list(full.get("client_tool_calls") or []) + if calls or client_calls: + call_source = "transcript" + notes.append(f"calls_from_rollout:{rollout}") + if full.get("final_text") and not parsed.get("final_text"): + parsed["final_text"] = full["final_text"] + notes.append(f"final_text_from_rollout:{rollout}") + if full.get("usage") and not parsed.get("usage"): + parsed["usage"] = full["usage"] else: - server_cmd, server_args = real_cmd[0], real_cmd[1:] - mcp_args = write_codex_mcp_override_args( - command=server_cmd, - args=server_args, - env=child_env, - server_name="plane", - ) - cmd: list[str] = [ - self.codex_bin, - "exec", - "--json", - "--skip-git-repo-check", - *mcp_args, - ] - if model: - cmd.extend(["-m", model]) - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd.append(full_prompt) - - t0 = time.perf_counter() - timeout_s = max(120, max_turns * 60) - try: - proc = self._runner( - cmd, - cwd=str(cwd), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "stream" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - experimental=True, - notes=notes, - ) - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - parsed = parse_codex_jsonl_events(stdout) - calls = list(parsed.get("calls") or []) - client_calls = list(parsed.get("client_tool_calls") or []) - call_source = "stream" - session_id = parsed.get("session_id") - - # Exact-match rollout only — never steal another parallel task's file. - need_rollout = (not calls and not client_calls) or not parsed.get("final_text") - if need_rollout and session_id: - rollout = find_codex_rollout(session_id) - if rollout is not None: - full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) - if not calls and not client_calls: - calls = list(full.get("calls") or []) - client_calls = list(full.get("client_tool_calls") or []) - if calls or client_calls: - call_source = "transcript" - notes.append(f"calls_from_rollout:{rollout}") - if full.get("final_text") and not parsed.get("final_text"): - parsed["final_text"] = full["final_text"] - notes.append(f"final_text_from_rollout:{rollout}") - if full.get("usage") and not parsed.get("usage"): - parsed["usage"] = full["usage"] - else: - notes.append("codex_rollout_unmatched") - elif need_rollout and not session_id: notes.append("codex_rollout_unmatched") - - if self.use_proxy: - calls, client_calls, proxy_src = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proxy_src == "proxy": - call_source = "proxy" - - if proc.returncode != 0: - notes.append(f"codex_exit={proc.returncode}") - - usage = parsed.get("usage") - usage_total = None - if isinstance(usage, dict): - usage_total = { - "input_tokens": usage.get("input_tokens"), - "output_tokens": usage.get("output_tokens"), - "cache_read_input_tokens": usage.get("cache_read_input_tokens"), - "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), - "total_input_tokens_including_cache": ( - int(usage.get("input_tokens") or 0) - + int(usage.get("cache_read_input_tokens") or 0) - + int(usage.get("cache_creation_input_tokens") or 0) - ), - "source": "codex_token_count", - } - - raw_ref = f"session:{session_id}" if session_id else None - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=parsed.get("final_text") or "", - usage=usage, - usage_total=usage_total, - stopped_reason=parsed.get("stopped_reason") or "end_turn", - raw_ref=raw_ref, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, # codex exec has no max-turns flag in --help - wall_time_s=round(wall, 3), - experimental=True, - notes=notes, - ) + elif need_rollout and not session_id: + notes.append("codex_rollout_unmatched") + + usage = parsed.get("usage") + usage_total = None + if isinstance(usage, dict): + usage_total = { + "input_tokens": usage.get("input_tokens"), + "output_tokens": usage.get("output_tokens"), + "cache_read_input_tokens": usage.get("cache_read_input_tokens"), + "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), + "total_input_tokens_including_cache": ( + int(usage.get("input_tokens") or 0) + + int(usage.get("cache_read_input_tokens") or 0) + + int(usage.get("cache_creation_input_tokens") or 0) + ), + "source": "codex_token_count", + } + + raw_ref = f"session:{session_id}" if session_id else None + return CliOutput( + calls=calls, + final_text=parsed.get("final_text") or "", + client_tool_calls=client_calls, + usage=usage, + usage_total=usage_total, + stopped_reason=parsed.get("stopped_reason") or "end_turn", + raw_ref=raw_ref, + call_source=call_source, + hit_max_turns=False, # codex exec has no max-turns flag in --help + ) __all__ = [ diff --git a/evals/drivers/opencode.py b/evals/drivers/opencode.py index a007919f..83b13865 100644 --- a/evals/drivers/opencode.py +++ b/evals/drivers/opencode.py @@ -4,21 +4,10 @@ import json import subprocess -import sys -import tempfile -import time from collections.abc import Callable from pathlib import Path -from typing import Any -from evals.drivers.base import REPO_ROOT, AgentRun -from evals.drivers.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - proxy_wrap_server_command, -) +from evals.drivers.cli import CliDriver, CliLaunch, CliOutput # OpenCode CLI — proxy-first # --------------------------------------------------------------------------- @@ -51,7 +40,7 @@ def write_opencode_mcp_config( path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") -class OpencodeCliDriver: +class OpencodeCliDriver(CliDriver): """Run tasks via ``opencode run`` (proxy-first call recording). Probed flags (2026-08-12): @@ -63,6 +52,11 @@ class OpencodeCliDriver: """ name = "opencode-cli" + run_notes = ("no_turn_cap",) + temp_dir_prefix = "plane-eval-opencode-" + temp_dir_in_cwd = True + exit_note_prefix = "opencode" + include_stderr_in_exit_note = True def __init__( self, @@ -75,142 +69,84 @@ def __init__( record_result_payloads: bool = False, ) -> None: self.opencode_bin = opencode_bin - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads - - def run_task( + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del task_cwd + # Project-local config avoids polluting the user's global config. + write_opencode_mcp_config( + temp_dir / "opencode.json", + command=server_command, + env=child_env, + server_name="plane", + ) + return CliLaunch(cwd=temp_dir) + + def build_command( self, prompt: str, - mcp_env: dict[str, str], + *, model: str | None, max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns, launch + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + command = [self.opencode_bin, "run", "--format", "json"] + if model: + command.extend(["-m", model]) + command.append(full_prompt) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - base_cwd = (cwd or REPO_ROOT).resolve() - notes: list[str] = ["no_turn_cap"] - t0 = time.perf_counter() - child_env_plane = {k: v for k, v in mcp_env.items() if k.startswith("PLANE_") or k in ("PATH", "HOME")} - - with tempfile.TemporaryDirectory(prefix="plane-eval-opencode-", dir=str(base_cwd)) as td: - # Project-local opencode.json so we do not pollute the user's global config. - proj = Path(td) - sidecar = proj / "proxy-sidecar.jsonl" - if self.server_command: - real_cmd = list(self.server_command) - else: - real_cmd = [self.python_bin, "-m", "plane_mcp", "stdio"] - if self.use_proxy: - launch = proxy_wrap_server_command( - real_cmd, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - ) - child_env_plane = ensure_proxy_pythonpath(child_env_plane) - else: - launch = real_cmd - write_opencode_mcp_config( - proj / "opencode.json", - command=launch, - env=child_env_plane, - server_name="plane", - ) - - full_prompt = prompt if not system else f"{system}\n\n{prompt}" - cmd: list[str] = [ - self.opencode_bin, - "run", - "--format", - "json", - ] - if model: - cmd.extend(["-m", model]) - cmd.append(full_prompt) - - timeout_s = max(120, max_turns * 60) - try: - proc = self._runner( - cmd, - cwd=str(proj), - capture_output=True, - text=True, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - t0 - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - calls_to: list[dict[str, Any]] = [] - client_to: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls_to, client_to, call_source = harvest_proxy_after_cli_timeout( - calls_to, client_to, sidecar, notes - ) - return AgentRun( - calls=calls_to, - client_tool_calls=client_to, - final_text="", - usage=None, - stopped_reason="timeout", - usage_scope="run", - call_source=call_source, - wall_time_s=round(wall, 3), - notes=notes, - ) - - wall = time.perf_counter() - t0 - stdout = proc.stdout or "" - stderr = proc.stderr or "" - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = "json" - if self.use_proxy: - calls, client_calls, call_source = apply_proxy_sidecar(calls, client_calls, sidecar, notes) - if proc.returncode != 0: - notes.append(f"opencode_exit={proc.returncode}") - if stderr.strip(): - notes.append(stderr.strip()[:500]) - - final_text = stdout.strip() - # JSONL events: concatenate text-ish fields best-effort. - if final_text and "\n" in final_text: - parts: list[str] = [] - for line in final_text.splitlines(): - line = line.strip() - if not line.startswith("{"): - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(row, dict): - for key in ("text", "message", "part", "delta"): - v = row.get(key) - if isinstance(v, str) and v.strip(): - parts.append(v) - if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): - parts.append(row["content"]) - if parts: - final_text = "\n".join(parts) - - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text=final_text, - usage=None, - stopped_reason="error" if proc.returncode else "end_turn", - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - notes=notes, - ) + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del task_cwd, max_turns + final_text = (proc.stdout or "").strip() + # JSONL events: concatenate text-ish fields best-effort. + if final_text and "\n" in final_text: + parts: list[str] = [] + for line in final_text.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + for key in ("text", "message", "part", "delta"): + value = row.get(key) + if isinstance(value, str) and value.strip(): + parts.append(value) + if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): + parts.append(row["content"]) + if parts: + final_text = "\n".join(parts) + + return CliOutput( + final_text=final_text, + stopped_reason="error" if proc.returncode else "end_turn", + ) __all__ = [ From 3ddebae9fc1ee6c53ea3d22f7276f377473263fe Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 10:29:31 +0530 Subject: [PATCH 10/93] Declare the result schema instead of assembling it by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A result row was an untyped dict of ~32 keys, built in pieces across runner.py and the driver row mapper, and read in report.py through some thirty separate .get() calls. Nothing declared the shape, so a new field had to be threaded through every construction site by hand — which is how result_tokens_estimated was added — and a mistyped key read as None and reported as zero. CallRecord and TaskResult now own one to_row serializer and one from_row reader, with an explicit schema version. Usage's short on-disk keys survive as part of that declared schema rather than a hand-maintained "legacy" shim; a docstring records that nothing in-repo reads them and they exist for humans and ad-hoc analysis of stored runs. Reading old files is the point, not a nicety: past batteries are what the harness is for. from_row loads pre-change rows, proven against real rows lifted out of battery4 and battery5 files rather than synthesized ones, and report output on those files is byte-identical to before this change. wall_time_s now means the same thing everywhere. Codex measured the CLI invocation while the other three included our own config setup, and the field is compared across drivers. It excludes setup for all of them, mirroring how the API driver times the agent loop alone; rows written earlier include a few milliseconds of setup for three drivers. A test now pins the template contract itself: a minimal subclass inherits proxy-first call counting and timeout harvesting without reimplementing either, which is the guarantee the base class exists to provide. Co-Authored-By: Claude Fable 5 --- evals/drivers/__init__.py | 5 + evals/drivers/api/backend.py | 21 +- evals/drivers/api/driver.py | 5 +- evals/drivers/base.py | 153 ++++---- evals/drivers/cli.py | 6 +- evals/drivers/codex.py | 1 - evals/report.py | 156 +++++---- evals/results.py | 390 +++++++++++++++++++++ evals/runner.py | 186 +++++----- tests/fixtures/evals_historical_rows.jsonl | 2 + tests/test_evals_api_driver.py | 8 +- tests/test_evals_hardening.py | 7 +- tests/test_evals_proxy.py | 95 +++++ tests/test_evals_report_ops.py | 60 +++- 14 files changed, 814 insertions(+), 281 deletions(-) create mode 100644 evals/results.py create mode 100644 tests/fixtures/evals_historical_rows.jsonl diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 49ada485..647c5e46 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -49,6 +49,7 @@ AgentDriver, AgentRun, agent_run_to_harness_dict, + agent_run_to_task_result, is_plane_mcp_tool, normalize_tool_call, split_plane_and_client_calls, @@ -81,6 +82,7 @@ proxy_wrap_server_command, wait_for_proxy_meta, ) +from evals.results import CallRecord, TaskResult # Registry # --------------------------------------------------------------------------- @@ -111,11 +113,14 @@ def get_driver(name: str, **kwargs: Any) -> AgentDriver: "AntigravityCliDriver", "ApiDriver", "ClaudeCliDriver", + "CallRecord", "CliDriver", "CodexCliDriver", "OpencodeCliDriver", "REPO_ROOT", + "TaskResult", "agent_run_to_harness_dict", + "agent_run_to_task_result", "apply_proxy_sidecar", "ensure_proxy_pythonpath", "find_claude_transcript", diff --git a/evals/drivers/api/backend.py b/evals/drivers/api/backend.py index a50b2fa5..22030830 100644 --- a/evals/drivers/api/backend.py +++ b/evals/drivers/api/backend.py @@ -7,6 +7,8 @@ from enum import Enum from typing import Any, Protocol +from evals.results import Usage + MODEL_TIERS = frozenset({"standard", "fast"}) @@ -37,25 +39,6 @@ class StopReason(str, Enum): UNKNOWN = "unknown" -@dataclass(frozen=True) -class Usage: - """Provider-neutral token usage for one model turn.""" - - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_input_tokens: int = 0 - cache_creation_input_tokens: int = 0 - - def to_legacy_dict(self) -> dict[str, int]: - """Return the established per-iteration JSONL shape.""" - return { - "in": self.input_tokens, - "out": self.output_tokens, - "cache_read": self.cache_read_input_tokens, - "cache_write": self.cache_creation_input_tokens, - } - - @dataclass(frozen=True) class ToolSpec: """A model-facing tool definition translated from MCP ``list_tools``.""" diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index a248fdd7..10f17cc4 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -24,6 +24,7 @@ create_backend, ) from evals.drivers.base import AgentRun +from evals.results import Usage from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens DEFAULT_MAX_TOKENS = 8192 @@ -206,7 +207,7 @@ async def _run_task( backend = self._make_backend(model) calls: list[dict[str, Any]] = [] pending_results: list[tuple[int, str]] = [] - usage_per_iteration: list[dict[str, int]] = [] + usage_per_iteration: list[Usage] = [] total_input_tokens = 0 total_output_tokens = 0 total_cache_read_input_tokens = 0 @@ -237,7 +238,7 @@ async def _run_task( stop_reason = turn.stop_reason provider_stop_reason = turn.provider_stop_reason if turn.usage is not None: - usage_per_iteration.append(turn.usage.to_legacy_dict()) + usage_per_iteration.append(turn.usage) total_input_tokens += turn.usage.input_tokens total_output_tokens += turn.usage.output_tokens total_cache_read_input_tokens += turn.usage.cache_read_input_tokens diff --git a/evals/drivers/base.py b/evals/drivers/base.py index cdfc51c3..1b604223 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any, Protocol +from evals.results import CallRecord, TaskResult, Usage from evals.token_counting import ( TOKEN_ESTIMATE_METHOD, count_result_text_tokens, @@ -31,7 +32,7 @@ class AgentRun: # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} calls: list[dict[str, Any]] final_text: str - usage: dict[str, Any] | None + usage: Usage | dict[str, Any] | None stopped_reason: str raw_ref: str | None = None # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics @@ -45,7 +46,7 @@ class AgentRun: wall_time_s: float = 0.0 experimental: bool = False notes: list[str] = field(default_factory=list) - usage_per_iteration: list[dict[str, int]] = field(default_factory=list) + usage_per_iteration: list[Usage] = field(default_factory=list) cum_input_tokens: int | None = None result_pair_mismatch: bool = False token_count_failures: int = 0 @@ -158,14 +159,14 @@ def split_plane_and_client_calls( return plane, client -def agent_run_to_harness_dict( +def agent_run_to_task_result( run: AgentRun, *, optimal: set[str], alternate: set[str], classify: Callable[[str, set[str], set[str]], str], -) -> dict[str, Any]: - """Map an ``AgentRun`` onto the dict shape expected by ``run_live`` rows. +) -> TaskResult: + """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. Only **plane** MCP tools are classified and counted in ``num_calls`` / mispick metrics. Client built-ins (``ToolSearch``, …) go to @@ -180,7 +181,7 @@ def agent_run_to_harness_dict( client_src = list(run.client_tool_calls) + client_extra is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" - calls: list[dict[str, Any]] = [] + calls: list[CallRecord] = [] local_token_count_failures = 0 for c in plane_src: tool = c.get("tool") or "" @@ -210,26 +211,25 @@ def agent_run_to_harness_dict( estimated = True count_method = TOKEN_ESTIMATE_METHOD - rec: dict[str, Any] = { - "tool": tool, - "class": classify(str(tool), optimal, alternate), - "args_chars": args_chars, - "result_tokens": result_tokens, - "result_chars": result_chars, - "result_kind": str(c.get("result_kind") or "text"), - "is_error": bool(c.get("is_error")), - "result_tokens_estimated": bool(estimated), - "result_token_count_method": str(count_method), - } - if c.get("duration_ms") is not None: - rec["duration_ms"] = c["duration_ms"] + rec = CallRecord( + tool=str(tool), + classification=classify(str(tool), optimal, alternate), + args_chars=args_chars, + result_tokens=result_tokens, + result_chars=result_chars, + result_kind=str(c.get("result_kind") or "text"), + is_error=bool(c.get("is_error")), + result_tokens_estimated=bool(estimated), + result_token_count_method=str(count_method), + duration_ms=c.get("duration_ms"), + ) # Action-dispatch surfaces: the action arg IS the second half of the # tool choice — keep it (args content is otherwise not persisted). if isinstance(args, dict) and isinstance(args.get("action"), str): - rec["action"] = args["action"] + rec.action = args["action"] calls.append(rec) - client_tool_calls: list[dict[str, Any]] = [] + client_tool_calls: list[CallRecord] = [] for c in client_src: tool = c.get("tool") or c.get("raw_tool") or "" args = c.get("args") or {} @@ -238,11 +238,11 @@ def agent_run_to_harness_dict( except Exception: args_chars = len(str(args)) client_tool_calls.append( - { - "tool": tool, - "args_chars": args_chars, - "raw_tool": c.get("raw_tool") or tool, - } + CallRecord( + tool=str(tool), + args_chars=args_chars, + raw_tool=str(c.get("raw_tool") or tool), + ) ) stop_reason = run.stopped_reason @@ -250,9 +250,9 @@ def agent_run_to_harness_dict( if hit_max: stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" - errored = sum(1 for c in calls if c.get("is_error")) - alternate_n = sum(1 for c in calls if c["class"] == "alternate") - out_of_set_n = sum(1 for c in calls if c["class"] == "out_of_set") + errored = sum(1 for c in calls if c.is_error) + alternate_n = sum(1 for c in calls if c.classification == "alternate") + out_of_set_n = sum(1 for c in calls if c.classification == "out_of_set") # CLI path: never write misleading cum_input_tokens from uncached-only field. # usage_total is driver-owned — do not re-derive it here (Claude vs Codex @@ -260,11 +260,11 @@ def agent_run_to_harness_dict( usage_total = run.usage_total if run.usage_per_iteration: - usage_per_iteration = [dict(item) for item in run.usage_per_iteration] + usage_per_iteration = list(run.usage_per_iteration) cum_input = ( run.cum_input_tokens if run.cum_input_tokens is not None - else sum(item.get("in", 0) for item in usage_per_iteration) + else sum(item.input_tokens for item in usage_per_iteration) ) cum_reason = None elif is_cli: @@ -273,7 +273,7 @@ def agent_run_to_harness_dict( "CLI driver: Claude usage.input_tokens is uncached-only; " "see usage_total (cache_read/cache_creation/output/cost) for run accounting" ) - usage_per_iteration: list[dict[str, int]] = [] + usage_per_iteration: list[Usage] = [] else: cum_input = 0 cum_reason = None @@ -281,7 +281,7 @@ def agent_run_to_harness_dict( if run.usage and run.usage_scope == "iteration": pass - estimated_states = [bool(c["result_tokens_estimated"]) for c in calls] + estimated_states = [bool(c.result_tokens_estimated) for c in calls] if estimated_states: result_tokens_estimated = any(estimated_states) result_tokens_mode = ( @@ -293,49 +293,61 @@ def agent_run_to_harness_dict( ) result_tokens_mode = "estimated" if result_tokens_estimated else "measured" - count_methods = {str(c["result_token_count_method"]) for c in calls} + count_methods = {str(c.result_token_count_method) for c in calls} if not count_methods: result_token_count_method = "none" elif len(count_methods) == 1: result_token_count_method = next(iter(count_methods)) else: result_token_count_method = "mixed" - result = { - "final_text": run.final_text, - "calls": calls, - "num_calls": len(calls), - "client_tool_calls": client_tool_calls, - "client_tool_call_count": len(client_tool_calls), - "errored_calls": errored, - "alternate_calls": alternate_n, - "out_of_set_calls": out_of_set_n, - "total_result_tokens": sum(int(c["result_tokens"]) for c in calls), - "usage_per_iteration": usage_per_iteration, - "cum_input_tokens": cum_input, - "cum_input_tokens_reason": cum_reason, - "wall_time_s": run.wall_time_s, - "stop_reason": stop_reason, - "provider_stop_reason": run.provider_stop_reason, - "hit_max_iterations": hit_max, - "result_pair_mismatch": run.result_pair_mismatch, - "token_count_failures": run.token_count_failures + local_token_count_failures, - "result_tokens_estimated": result_tokens_estimated, - "result_tokens_mode": result_tokens_mode, - "result_token_count_method": result_token_count_method, - "usage_scope": run.usage_scope, - "call_source": run.call_source, - "driver_raw_ref": run.raw_ref, - "driver_notes": list(run.notes), - "usage": run.usage, - "usage_total": usage_total, - } - if run.provider is not None: - result["provider"] = run.provider - if run.model is not None: - result["model"] = run.model - if run.requested_model is not None: - result["requested_model"] = run.requested_model - return result + return TaskResult( + final_text=run.final_text, + calls=calls, + num_calls=len(calls), + client_tool_calls=client_tool_calls, + client_tool_call_count=len(client_tool_calls), + errored_calls=errored, + alternate_calls=alternate_n, + out_of_set_calls=out_of_set_n, + total_result_tokens=sum(int(c.result_tokens or 0) for c in calls), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=cum_input, + cum_input_tokens_reason=cum_reason, + wall_time_s=run.wall_time_s, + stop_reason=stop_reason, + provider_stop_reason=run.provider_stop_reason, + hit_max_iterations=hit_max, + result_pair_mismatch=run.result_pair_mismatch, + token_count_failures=run.token_count_failures + local_token_count_failures, + result_tokens_estimated=result_tokens_estimated, + result_tokens_mode=result_tokens_mode, + result_token_count_method=result_token_count_method, + usage_scope=run.usage_scope, + call_source=run.call_source, + driver_raw_ref=run.raw_ref, + driver_notes=list(run.notes), + usage=run.usage, + usage_total=usage_total, + provider=run.provider, + model=run.model, + requested_model=run.requested_model, + ) + + +def agent_run_to_harness_dict( + run: AgentRun, + *, + optimal: set[str], + alternate: set[str], + classify: Callable[[str, set[str], set[str]], str], +) -> dict[str, Any]: + """Compatibility wrapper returning the public persisted-row dictionary.""" + return agent_run_to_task_result( + run, + optimal=optimal, + alternate=alternate, + classify=classify, + ).to_row() __all__ = [ @@ -343,6 +355,7 @@ def agent_run_to_harness_dict( "AgentRun", "AgentDriver", "agent_run_to_harness_dict", + "agent_run_to_task_result", "is_plane_mcp_tool", "normalize_tool_call", "split_plane_and_client_calls", diff --git a/evals/drivers/cli.py b/evals/drivers/cli.py index 960a8157..74a28684 100644 --- a/evals/drivers/cli.py +++ b/evals/drivers/cli.py @@ -59,7 +59,6 @@ class CliDriver(ABC): run_notes: tuple[str, ...] = () temp_dir_prefix = "plane-eval-cli-" temp_dir_in_cwd = False - include_setup_in_wall_time = True exit_note_prefix: str | None = None include_stderr_in_exit_note = False @@ -163,7 +162,6 @@ def run_task( task_cwd = (cwd or REPO_ROOT).resolve() notes = list(self.run_notes) self.validate_run() - started_at = time.perf_counter() if self.include_setup_in_wall_time else None temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None with tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td: @@ -198,9 +196,9 @@ def run_task( system=system, launch=launch, ) - if started_at is None: - started_at = time.perf_counter() timeout_s = max(120, max_turns * 60) + # Persisted schema v1 defines wall time as the CLI invocation only. + started_at = time.perf_counter() try: proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) diff --git a/evals/drivers/codex.py b/evals/drivers/codex.py index a29be44f..28d99a46 100644 --- a/evals/drivers/codex.py +++ b/evals/drivers/codex.py @@ -237,7 +237,6 @@ class CodexCliDriver(CliDriver): default_call_source = "stream" run_notes = ("experimental:codex-cli",) temp_dir_prefix = "plane-eval-codex-" - include_setup_in_wall_time = False exit_note_prefix = "codex" def __init__( diff --git a/evals/report.py b/evals/report.py index fceb15df..fe11e477 100644 --- a/evals/report.py +++ b/evals/report.py @@ -17,10 +17,16 @@ from pathlib import Path from typing import Any, Literal +from evals.results import TaskResult from evals.tasks import TASKS_BY_ID DedupeMode = Literal["latest", "none"] ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] +ResultRow = TaskResult | dict[str, Any] + + +def _task_result(row: ResultRow) -> TaskResult: + return row if isinstance(row, TaskResult) else TaskResult.from_row(row) def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: @@ -83,15 +89,19 @@ def _iqr(xs: list[float]) -> tuple[float | None, float | None, float | None]: return (_percentile(xs, 0.25), _median(xs), _percentile(xs, 0.75)) -def result_tokens_mode(rows: list[dict[str, Any]]) -> ResultTokensMode: +def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: """Classify token counts without treating unmarked legacy data as measured.""" labels: set[str] = set() - for row in rows: - row_estimated = row.get("result_tokens_estimated") - for call in row.get("calls") or []: - if call.get("result_tokens") is None: + for raw_row in rows: + row = _task_result(raw_row) + for call in row.calls: + if call.result_tokens is None: continue - estimated = call.get("result_tokens_estimated", row_estimated) + estimated = ( + call.result_tokens_estimated + if call.result_tokens_estimated is not None + else row.result_tokens_estimated + ) if estimated is True: labels.add("estimated") elif estimated is False: @@ -109,42 +119,45 @@ def result_tokens_mode(rows: list[dict[str, Any]]) -> ResultTokensMode: return "mixed" -def is_meta_row(row: dict[str, Any]) -> bool: +def is_meta_row(row: ResultRow) -> bool: """True for run-header meta lines (or any row without a task_id).""" + if isinstance(row, TaskResult): + return not row.task_id if row.get("row_type") == "meta": return True return row.get("task_id") is None -def is_infra_error_row(row: dict[str, Any]) -> bool: +def is_infra_error_row(row: ResultRow) -> bool: """True when a row failed for infrastructure reasons (seed/cli/api), not task verify. Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, ``infra_api``, ``infra_sdk``, …) is excluded from success-rate denominators. """ - ec = row.get("error_class") + ec = _task_result(row).error_class return isinstance(ec, str) and ec.startswith("infra_") -def dedupe_rows_latest(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: +def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: """Keep only the last row per (task_id, rep, surface); preserve key insertion order.""" - latest: dict[tuple[Any, Any, Any], dict[str, Any]] = {} - order: list[tuple[Any, Any, Any]] = [] - for r in rows: - key = (r.get("task_id"), r.get("rep"), r.get("surface")) + latest: dict[tuple[str, int, str], TaskResult] = {} + order: list[tuple[str, int, str]] = [] + for raw_row in rows: + row = _task_result(raw_row) + key = (row.task_id, row.rep, row.surface) if key not in latest: order.append(key) - latest[key] = r + latest[key] = row return [latest[k] for k in order] -def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[dict[str, Any]]: +def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: """Load JSONL data rows (skip meta / missing task_id). Default ``dedupe="latest"`` keeps the last row per (task_id, rep, surface) so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. """ - rows: list[dict[str, Any]] = [] + rows: list[TaskResult] = [] with path.open(encoding="utf-8") as fh: for line_no, line in enumerate(fh, start=1): line = line.strip() @@ -160,13 +173,13 @@ def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[dict[str, An continue if not isinstance(row, dict) or is_meta_row(row): continue - rows.append(row) + rows.append(TaskResult.from_row(row)) if dedupe == "latest": return dedupe_rows_latest(rows) # Forensics: warn on duplicates but keep all. - seen_keys: set[tuple[Any, Any, Any]] = set() + seen_keys: set[tuple[str, int, str]] = set() for r in rows: - key = (r.get("task_id"), r.get("rep"), r.get("surface")) + key = (r.task_id, r.rep, r.surface) if key in seen_keys: print( f"warning: {path}: duplicate (task_id, rep, surface)={key} " @@ -178,7 +191,7 @@ def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[dict[str, An return rows -def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: +def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: """Aggregate per-task metrics. Rows with ``error_class`` starting ``infra_`` are excluded from success-rate @@ -186,22 +199,23 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: dict under the special key ``_meta``). Other non-null ``error`` rows remain harness errors (excluded from success, counted in ``harness_err``). """ - by_task: dict[str, list[dict[str, Any]]] = defaultdict(list) + by_task: dict[str, list[TaskResult]] = defaultdict(list) harness_err_by_task: dict[str, int] = defaultdict(int) infra_err_by_task: dict[str, int] = defaultdict(int) infra_errors = 0 - for r in rows: + for raw_row in rows: + r = _task_result(raw_row) if is_meta_row(r): continue - tid = r["task_id"] + tid = r.task_id if is_infra_error_row(r): infra_errors += 1 infra_err_by_task[tid] += 1 continue # infra seed/cli — excluded from success aggregates - if r.get("error"): + if r.error: harness_err_by_task[tid] += 1 continue # harness/API errors excluded from success/medians (F4) - if r.get("skipped"): + if r.skipped: continue # skipped rows are excluded from success denominators by_task[tid].append(r) @@ -214,11 +228,11 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: for task_id in all_task_ids: trs = by_task.get(task_id, []) n = len(trs) - k = sum(1 for r in trs if r.get("success")) + k = sum(1 for r in trs if r.success) total_k += k total_n += n lo, hi = wilson_interval(k, n) if n else (0.0, 0.0) - calls = [float(r.get("num_calls") or 0) for r in trs] + calls = [float(r.num_calls) for r in trs] q1, med_calls, q3 = _iqr(calls) min_calls = min(calls) if calls else None max_calls = max(calls) if calls else None @@ -228,16 +242,16 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: errored = 0 result_tokens: list[float] = [] for r in trs: - for c in r.get("calls") or []: + for c in r.calls: total_calls += 1 - if c.get("class") in ("alternate", "out_of_set"): + if c.classification in ("alternate", "out_of_set"): mispick += 1 - if c.get("is_error"): + if c.is_error: errored += 1 - if c.get("result_tokens") is not None: - result_tokens.append(float(c["result_tokens"])) - capped = sum(1 for r in trs if r.get("hit_max_iterations") or r.get("stop_reason") == "max_tokens") - cum_inputs = [float(r.get("cum_input_tokens") or 0) for r in trs] + if c.result_tokens is not None: + result_tokens.append(float(c.result_tokens)) + capped = sum(1 for r in trs if r.hit_max_iterations or r.stop_reason == "max_tokens") + cum_inputs = [float(r.cum_input_tokens or 0) for r in trs] out[task_id] = { "n": n, "k": k, @@ -366,8 +380,8 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: def ab_compare( - rows_a: list[dict[str, Any]], - rows_b: list[dict[str, Any]], + rows_a: list[ResultRow], + rows_b: list[ResultRow], ) -> dict[str, Any]: """Compare two result sets: paired call-count deltas + success rates. @@ -378,14 +392,15 @@ def ab_compare( sum_a = summarize(rows_a) sum_b = summarize(rows_b) - def _success_rows_by_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - out: dict[str, dict[str, Any]] = {} - for r in rows: - if is_meta_row(r) or is_infra_error_row(r) or r.get("error") or r.get("skipped"): + def _success_rows_by_task(rows: list[ResultRow]) -> dict[str, TaskResult]: + out: dict[str, TaskResult] = {} + for raw_row in rows: + r = _task_result(raw_row) + if is_meta_row(r) or is_infra_error_row(r) or r.error or r.skipped: continue - if not r.get("success"): + if not r.success: continue - tid = str(r["task_id"]) + tid = r.task_id out[tid] = r # last wins (dedupe already applied) return out @@ -395,8 +410,8 @@ def _success_rows_by_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any deltas: list[float] = [] per_task: list[dict[str, Any]] = [] for tid in shared: - ca = float(sa[tid].get("num_calls") or 0) - cb = float(sb[tid].get("num_calls") or 0) + ca = float(sa[tid].num_calls) + cb = float(sb[tid].num_calls) d = cb - ca # B − A (negative = B fewer calls = better if lower is better) deltas.append(d) per_task.append({"task_id": tid, "calls_a": ca, "calls_b": cb, "delta": d}) @@ -459,21 +474,21 @@ def _task_sort_key(tid: str) -> tuple[str, int]: return (tid[0] if tid else "", int(digits) if digits else 0) -def format_surface_cell(row: dict[str, Any] | None) -> str: +def format_surface_cell(row: ResultRow | None) -> str: """Cell for multi-surface table: '✅ Nc/Mmp', 'skip', 'ERR', or '—'.""" if row is None: return "—" - if row.get("skipped"): + result = _task_result(row) + if result.skipped: return "skip" - if row.get("error") or is_infra_error_row(row): + if result.error or is_infra_error_row(result): return "ERR" - ok = "✅" if row.get("success") else "❌" - n_calls = row.get("num_calls") - n_calls_s = str(n_calls) if n_calls is not None else "?" - if row.get("classification") == "external": + ok = "✅" if result.success else "❌" + n_calls_s = str(result.num_calls) + if result.classification == "external": return f"{ok} {n_calls_s}c" - alt = row.get("alternate_calls") - oos = row.get("out_of_set_calls") + alt = result.alternate_calls + oos = result.out_of_set_calls # None counters (external nulling) → omit mispick suffix. if alt is None and oos is None: return f"{ok} {n_calls_s}c" @@ -484,7 +499,7 @@ def format_surface_cell(row: dict[str, Any] | None) -> str: def build_multi_surface_table( - file_rows: list[tuple[str, list[dict[str, Any]]]], + file_rows: list[tuple[str, list[ResultRow]]], ) -> dict[str, Any]: """Build a per-task × per-surface grid from labeled row sets. @@ -493,20 +508,21 @@ def build_multi_surface_table( For each column, the latest row per task_id is used (rep-agnostic: last wins). """ columns: list[str] = [] - by_col: dict[str, dict[str, dict[str, Any]]] = {} + by_col: dict[str, dict[str, TaskResult]] = {} for label, rows in file_rows: columns.append(label) - col_map: dict[str, dict[str, Any]] = {} - for r in rows: - if is_meta_row(r): + col_map: dict[str, TaskResult] = {} + for raw_row in rows: + if is_meta_row(raw_row): continue - tid = str(r["task_id"]) + r = _task_result(raw_row) + tid = r.task_id col_map[tid] = r # last wins by_col[label] = col_map all_tasks = sorted({t for m in by_col.values() for t in m}, key=_task_sort_key) cells: dict[str, dict[str, str]] = {} - raw: dict[str, dict[str, dict[str, Any] | None]] = {} + raw: dict[str, dict[str, TaskResult | None]] = {} for tid in all_tasks: cells[tid] = {} raw[tid] = {} @@ -525,18 +541,18 @@ def build_multi_surface_table( if is_infra_error_row(r): infra += 1 continue - if r.get("error"): + if r.error: continue - if r.get("skipped"): + if r.skipped: continue run += 1 - if r.get("success"): + if r.success: succ += 1 - calls += int(r.get("num_calls") or 0) - if r.get("classification") == "external": + calls += r.num_calls + if r.classification == "external": mispick_comparable = False else: - alt, oos = r.get("alternate_calls"), r.get("out_of_set_calls") + alt, oos = r.alternate_calls, r.out_of_set_calls if alt is None and oos is None: mispick_comparable = False else: @@ -600,11 +616,11 @@ def _prompt_snip(tid: str) -> str: return "\n".join(lines) + "\n" -def _surface_label_for_file(path: Path, rows: list[dict[str, Any]]) -> str: +def _surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: """Pick a column label from the file's dominant surface field, else stem.""" counts: dict[str, int] = defaultdict(int) for r in rows: - s = r.get("surface") + s = r.surface if s: counts[str(s)] += 1 if counts: @@ -651,7 +667,7 @@ def main(argv: list[str] | None = None) -> int: if len(paths) < 1: print("error: --table requires at least one JSONL", file=sys.stderr) return 2 - labeled: list[tuple[str, list[dict[str, Any]]]] = [] + labeled: list[tuple[str, list[TaskResult]]] = [] used_labels: set[str] = set() for path in paths: rows = load_rows(path, dedupe=dedupe) diff --git a/evals/results.py b/evals/results.py new file mode 100644 index 00000000..1a6c60a0 --- /dev/null +++ b/evals/results.py @@ -0,0 +1,390 @@ +"""Declared persisted schema for eval task-result JSONL rows.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +RESULT_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True, slots=True) +class Usage: + """Provider-neutral token usage for one model turn.""" + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_input_tokens: int = 0 + cache_creation_input_tokens: int = 0 + + +@dataclass(slots=True) +class CallRecord: + """Persisted metrics for one Plane or client tool call.""" + + tool: str + classification: str | None = None + args_chars: int = 0 + result_tokens: int | None = None + result_chars: int = 0 + result_kind: str = "text" + is_error: bool = False + result_tokens_estimated: bool | None = None + result_token_count_method: str | None = None + duration_ms: float | int | None = None + action: str | None = None + raw_tool: str | None = None + result_tokens_skipped: str | None = None + + +@dataclass(slots=True) +class TaskResult: + """One task repetition and the complete persisted row schema. + + ``schema_version=0`` identifies rows written before this type existed; + :meth:`from_row` supplies defaults for every field added since. Version 1 + defines ``wall_time_s`` as CLI invocation time only, excluding harness-owned + config/temp-directory setup. Pre-versioned Claude, Antigravity, and OpenCode + rows include a few milliseconds of that setup; Codex and API rows already + used invocation/agent-loop timing. + """ + + schema_version: int = RESULT_SCHEMA_VERSION + run_id: str = "" + ts: str = "" + git_sha: str = "" + battery: str = "" + surface: str = "" + driver: str = "" + provider: str | None = None + classification: str = "" + model: str | None = None + requested_model: str | None = None + requested_tier: str | None = None + resolved_model: str | None = None + task_id: str = "" + author: str = "" + rep: int = 0 + success: bool = False + verify_note: str = "" + skipped: str | None = None + error: str | None = None + error_class: str | None = None + final_text: str = "" + stop_reason: str | None = None + provider_stop_reason: str | None = None + hit_max_iterations: bool = False + result_pair_mismatch: bool = False + token_count_failures: int = 0 + result_tokens_estimated: bool | None = None + calls: list[CallRecord] = field(default_factory=list) + num_calls: int = 0 + errored_calls: int = 0 + alternate_calls: int | None = 0 + out_of_set_calls: int | None = 0 + total_result_tokens: int = 0 + usage_per_iteration: list[Usage] = field(default_factory=list) + cum_input_tokens: int | None = 0 + cum_input_tokens_reason: str | None = None + wall_time_s: float = 0.0 + client_tool_calls: list[CallRecord] = field(default_factory=list) + client_tool_call_count: int = 0 + result_tokens_mode: str | None = None + result_token_count_method: str | None = None + usage_scope: str | None = None + call_source: str | None = None + driver_raw_ref: str | None = None + driver_notes: list[str] = field(default_factory=list) + usage: Usage | dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + result_tokens_skipped_reason: str | None = None + + def apply_agent_result(self, agent: TaskResult) -> None: + """Copy the driver-owned portion of an agent result onto this task row.""" + self.final_text = agent.final_text + self.stop_reason = agent.stop_reason + self.provider_stop_reason = agent.provider_stop_reason + self.hit_max_iterations = agent.hit_max_iterations + self.result_pair_mismatch = agent.result_pair_mismatch + self.token_count_failures = agent.token_count_failures + self.result_tokens_estimated = agent.result_tokens_estimated + self.calls = agent.calls + self.num_calls = agent.num_calls + self.errored_calls = agent.errored_calls + self.alternate_calls = agent.alternate_calls + self.out_of_set_calls = agent.out_of_set_calls + self.total_result_tokens = agent.total_result_tokens + self.usage_per_iteration = agent.usage_per_iteration + self.cum_input_tokens = agent.cum_input_tokens + self.cum_input_tokens_reason = agent.cum_input_tokens_reason + self.wall_time_s = agent.wall_time_s + self.client_tool_calls = agent.client_tool_calls + self.client_tool_call_count = agent.client_tool_call_count + self.result_tokens_mode = agent.result_tokens_mode + self.result_token_count_method = agent.result_token_count_method + self.usage_scope = agent.usage_scope + self.call_source = agent.call_source + self.driver_raw_ref = agent.driver_raw_ref + self.driver_notes = agent.driver_notes + self.usage = agent.usage + self.usage_total = agent.usage_total + if agent.provider is not None: + self.provider = agent.provider + if agent.model is not None: + self.model = agent.model + if agent.requested_model is not None: + self.requested_model = agent.requested_model + + def to_row(self) -> dict[str, Any]: + """Serialize the versioned persisted JSONL row schema. + + Per-iteration usage deliberately retains the established short keys + ``in``, ``out``, ``cache_read``, and ``cache_write``. Nothing in this + repository reads those keys; they are archival data for humans and + ad-hoc analysis, so the on-disk spelling remains stable here. + """ + + def usage_row(item: Usage) -> dict[str, int]: + return { + "in": item.input_tokens, + "out": item.output_tokens, + "cache_read": item.cache_read_input_tokens, + "cache_write": item.cache_creation_input_tokens, + } + + calls: list[dict[str, Any]] = [] + for call in self.calls: + item: dict[str, Any] = { + "tool": call.tool, + "class": call.classification, + "args_chars": call.args_chars, + "result_tokens": call.result_tokens, + "result_chars": call.result_chars, + "result_kind": call.result_kind, + "is_error": call.is_error, + "result_tokens_estimated": call.result_tokens_estimated, + "result_token_count_method": call.result_token_count_method, + } + if call.duration_ms is not None: + item["duration_ms"] = call.duration_ms + if call.action is not None: + item["action"] = call.action + if call.result_tokens_skipped is not None: + item["result_tokens_skipped"] = call.result_tokens_skipped + calls.append(item) + + client_calls = [ + { + "tool": call.tool, + "args_chars": call.args_chars, + "raw_tool": call.raw_tool or call.tool, + } + for call in self.client_tool_calls + ] + row: dict[str, Any] = { + "schema_version": self.schema_version, + "run_id": self.run_id, + "ts": self.ts, + "git_sha": self.git_sha, + "battery": self.battery, + "surface": self.surface, + "driver": self.driver, + "provider": self.provider, + "classification": self.classification, + "model": self.model, + "requested_model": self.requested_model, + "requested_tier": self.requested_tier, + "resolved_model": self.resolved_model, + "task_id": self.task_id, + "author": self.author, + "rep": self.rep, + "success": self.success, + "verify_note": self.verify_note, + "skipped": self.skipped, + "error": self.error, + "error_class": self.error_class, + "final_text": self.final_text, + "stop_reason": self.stop_reason, + "provider_stop_reason": self.provider_stop_reason, + "hit_max_iterations": self.hit_max_iterations, + "result_pair_mismatch": self.result_pair_mismatch, + "token_count_failures": self.token_count_failures, + "result_tokens_estimated": self.result_tokens_estimated, + "calls": calls, + "num_calls": self.num_calls, + "errored_calls": self.errored_calls, + "alternate_calls": self.alternate_calls, + "out_of_set_calls": self.out_of_set_calls, + "total_result_tokens": self.total_result_tokens, + "usage_per_iteration": [usage_row(item) for item in self.usage_per_iteration], + "cum_input_tokens": self.cum_input_tokens, + "cum_input_tokens_reason": self.cum_input_tokens_reason, + "wall_time_s": self.wall_time_s, + "client_tool_calls": client_calls, + "client_tool_call_count": self.client_tool_call_count, + "result_tokens_mode": self.result_tokens_mode, + "result_token_count_method": self.result_token_count_method, + "usage_scope": self.usage_scope, + "call_source": self.call_source, + "driver_raw_ref": self.driver_raw_ref, + "driver_notes": list(self.driver_notes), + "usage": usage_row(self.usage) if isinstance(self.usage, Usage) else self.usage, + "usage_total": self.usage_total, + } + if self.result_tokens_skipped_reason is not None: + row["result_tokens_skipped_reason"] = self.result_tokens_skipped_reason + return row + + @classmethod + def from_row(cls, row: dict[str, Any]) -> TaskResult: + """Read current or pre-versioned persisted rows with stable defaults.""" + raw_calls = row.get("calls") if isinstance(row.get("calls"), list) else [] + calls: list[CallRecord] = [] + for raw in raw_calls: + if not isinstance(raw, dict): + continue + calls.append( + CallRecord( + tool=str(raw.get("tool") or ""), + classification=(str(raw["class"]) if raw.get("class") is not None else None), + args_chars=int(raw.get("args_chars") or 0), + result_tokens=(int(raw["result_tokens"]) if raw.get("result_tokens") is not None else None), + result_chars=int(raw.get("result_chars") or 0), + result_kind=str(raw.get("result_kind") or "text"), + is_error=bool(raw.get("is_error")), + result_tokens_estimated=( + bool(raw["result_tokens_estimated"]) if raw.get("result_tokens_estimated") is not None else None + ), + result_token_count_method=( + str(raw["result_token_count_method"]) + if raw.get("result_token_count_method") is not None + else None + ), + duration_ms=raw.get("duration_ms"), + action=(str(raw["action"]) if raw.get("action") is not None else None), + result_tokens_skipped=( + str(raw["result_tokens_skipped"]) if raw.get("result_tokens_skipped") is not None else None + ), + ) + ) + + raw_client_calls = row.get("client_tool_calls") if isinstance(row.get("client_tool_calls"), list) else [] + client_calls: list[CallRecord] = [] + for raw in raw_client_calls: + if not isinstance(raw, dict): + continue + client_calls.append( + CallRecord( + tool=str(raw.get("tool") or raw.get("raw_tool") or ""), + args_chars=int(raw.get("args_chars") or 0), + raw_tool=str(raw.get("raw_tool") or raw.get("tool") or ""), + ) + ) + + raw_usage = row.get("usage_per_iteration") + usage_per_iteration: list[Usage] = [] + if isinstance(raw_usage, list): + for item in raw_usage: + if not isinstance(item, dict): + continue + usage_per_iteration.append( + Usage( + input_tokens=int(item.get("in") or 0), + output_tokens=int(item.get("out") or 0), + cache_read_input_tokens=int(item.get("cache_read") or 0), + cache_creation_input_tokens=int(item.get("cache_write") or 0), + ) + ) + + alternate_default = sum(1 for call in calls if call.classification == "alternate") + out_of_set_default = sum(1 for call in calls if call.classification == "out_of_set") + return cls( + schema_version=int(row.get("schema_version") or 0), + run_id=str(row.get("run_id") or ""), + ts=str(row.get("ts") or ""), + git_sha=str(row.get("git_sha") or ""), + battery=str(row.get("battery") or ""), + surface=str(row.get("surface") or ""), + driver=str(row.get("driver") or ""), + provider=(str(row["provider"]) if row.get("provider") is not None else None), + classification=str(row.get("classification") or ""), + model=(str(row["model"]) if row.get("model") is not None else None), + requested_model=(str(row["requested_model"]) if row.get("requested_model") is not None else None), + requested_tier=(str(row["requested_tier"]) if row.get("requested_tier") is not None else None), + resolved_model=(str(row["resolved_model"]) if row.get("resolved_model") is not None else None), + task_id=str(row.get("task_id") or ""), + author=str(row.get("author") or ""), + rep=int(row.get("rep") or 0), + success=bool(row.get("success")), + verify_note=str(row.get("verify_note") or ""), + skipped=(str(row["skipped"]) if row.get("skipped") is not None else None), + error=(str(row["error"]) if row.get("error") is not None else None), + error_class=(str(row["error_class"]) if row.get("error_class") is not None else None), + final_text=str(row.get("final_text") or ""), + stop_reason=(str(row["stop_reason"]) if row.get("stop_reason") is not None else None), + provider_stop_reason=( + str(row["provider_stop_reason"]) if row.get("provider_stop_reason") is not None else None + ), + hit_max_iterations=bool(row.get("hit_max_iterations")), + result_pair_mismatch=bool(row.get("result_pair_mismatch")), + token_count_failures=int(row.get("token_count_failures") or 0), + result_tokens_estimated=( + bool(row["result_tokens_estimated"]) if row.get("result_tokens_estimated") is not None else None + ), + calls=calls, + num_calls=int(row.get("num_calls") if row.get("num_calls") is not None else len(calls)), + errored_calls=int( + row.get("errored_calls") + if row.get("errored_calls") is not None + else sum(1 for call in calls if call.is_error) + ), + alternate_calls=( + int(row["alternate_calls"]) + if row.get("alternate_calls") is not None + else None + if "alternate_calls" in row + else alternate_default + ), + out_of_set_calls=( + int(row["out_of_set_calls"]) + if row.get("out_of_set_calls") is not None + else None + if "out_of_set_calls" in row + else out_of_set_default + ), + total_result_tokens=int( + row.get("total_result_tokens") + if row.get("total_result_tokens") is not None + else sum(call.result_tokens or 0 for call in calls) + ), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=(int(row["cum_input_tokens"]) if row.get("cum_input_tokens") is not None else None), + cum_input_tokens_reason=( + str(row["cum_input_tokens_reason"]) if row.get("cum_input_tokens_reason") is not None else None + ), + wall_time_s=float(row.get("wall_time_s") or 0.0), + client_tool_calls=client_calls, + client_tool_call_count=int( + row.get("client_tool_call_count") + if row.get("client_tool_call_count") is not None + else len(client_calls) + ), + result_tokens_mode=(str(row["result_tokens_mode"]) if row.get("result_tokens_mode") is not None else None), + result_token_count_method=( + str(row["result_token_count_method"]) if row.get("result_token_count_method") is not None else None + ), + usage_scope=(str(row["usage_scope"]) if row.get("usage_scope") is not None else None), + call_source=(str(row["call_source"]) if row.get("call_source") is not None else None), + driver_raw_ref=(str(row["driver_raw_ref"]) if row.get("driver_raw_ref") is not None else None), + driver_notes=[str(item) for item in row.get("driver_notes") or []], + usage=row.get("usage") if isinstance(row.get("usage"), dict) else None, + usage_total=(row.get("usage_total") if isinstance(row.get("usage_total"), dict) else None), + result_tokens_skipped_reason=( + str(row["result_tokens_skipped_reason"]) + if row.get("result_tokens_skipped_reason") is not None + else None + ), + ) + + +__all__ = ["RESULT_SCHEMA_VERSION", "CallRecord", "TaskResult", "Usage"] diff --git a/evals/runner.py b/evals/runner.py index dfde1a92..91a44172 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -14,10 +14,11 @@ from evals.drivers import ( KNOWN_DRIVERS, - agent_run_to_harness_dict, + agent_run_to_task_result, get_driver, ) from evals.drivers.api import MODEL_TIERS +from evals.results import RESULT_SCHEMA_VERSION, TaskResult from evals.seed import make_plane_client, seed, teardown from evals.tasks import ( PromptBindError, @@ -94,7 +95,7 @@ def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = No return env -def should_skip_resume_row(row: dict[str, Any]) -> bool: +def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: """Return True if a prior row is a completed result that resume should skip. Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. @@ -102,10 +103,11 @@ def should_skip_resume_row(row: dict[str, Any]) -> bool: surface/plan skips are stable outcomes, not infra failures). Pure function — unit-tested without the live battery. """ - ec = row.get("error_class") + result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) + ec = result.error_class if isinstance(ec, str) and ec.startswith("infra_"): return False - if row.get("error") is not None: + if result.error is not None: return False return True @@ -151,9 +153,9 @@ def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: return False -def _timeout_error_message(agent: dict[str, Any]) -> str: +def _timeout_error_message(agent: TaskResult) -> str: """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" - for note in agent.get("driver_notes") or []: + for note in agent.driver_notes: if isinstance(note, str) and note.startswith("timeout after"): return note return "timeout" @@ -225,13 +227,12 @@ def load_resume_skip_keys( # Meta / header rows: checked above, not part of resume key set. if is_meta_or_non_task_row(row): continue - tid = row.get("task_id") - rep = row.get("rep") - if tid is None or rep is None: + result = TaskResult.from_row(row) + if not result.task_id: continue - key = (str(tid), int(rep)) + key = (result.task_id, result.rep) seen.add(key) - if should_skip_resume_row(row): + if should_skip_resume_row(result): skip_keys.add(key) else: # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. @@ -256,6 +257,7 @@ def make_run_meta_row( ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" return { + "schema_version": RESULT_SCHEMA_VERSION, "row_type": "meta", "run_id": run_id, "surface": surface, @@ -292,7 +294,7 @@ async def run_agent_task_via_driver( optimal_tools: set[str] | None = None, alternate_tools: set[str] | None = None, server_env: dict[str, str] | None = None, -) -> dict[str, Any]: +) -> TaskResult: """Run one task through the selected AgentDriver.""" project_name = ctx["project_name"] system = _system_preamble(workspace_slug, project_name) @@ -312,7 +314,7 @@ async def run_agent_task_via_driver( system=system, cwd=Path(__file__).resolve().parent.parent, ) - return agent_run_to_harness_dict( + return agent_run_to_task_result( agent_run, optimal=optimal, alternate=alternate, @@ -334,45 +336,24 @@ def _base_row( rep: int, battery: str, classification: str, -) -> dict[str, Any]: - return { - "run_id": run_id, - "ts": datetime.now(timezone.utc).isoformat(), - "git_sha": git_sha, - "battery": battery, - "surface": surface, - "driver": driver_name, - "provider": provider, - "classification": classification, - "model": model_id, - "requested_model": model_request, - "requested_tier": requested_tier, - "resolved_model": model_id, - "task_id": task["id"], - "author": task_author(task), - "rep": rep, - "success": False, - "verify_note": "", - "skipped": None, - "error": None, - "error_class": None, - "final_text": "", - "stop_reason": None, - "provider_stop_reason": None, - "hit_max_iterations": False, - "result_pair_mismatch": False, - "token_count_failures": 0, - "result_tokens_estimated": None, - "calls": [], - "num_calls": 0, - "errored_calls": 0, - "alternate_calls": 0, - "out_of_set_calls": 0, - "total_result_tokens": 0, - "usage_per_iteration": [], - "cum_input_tokens": 0, - "wall_time_s": 0.0, - } +) -> TaskResult: + return TaskResult( + run_id=run_id, + ts=datetime.now(timezone.utc).isoformat(), + git_sha=git_sha, + battery=battery, + surface=surface, + driver=driver_name, + provider=provider, + classification=classification, + model=model_id, + requested_model=model_request, + requested_tier=requested_tier, + resolved_model=model_id, + task_id=str(task["id"]), + author=task_author(task), + rep=rep, + ) async def run_live( @@ -510,8 +491,8 @@ async def _run_tasks() -> None: # Surface-unsupported tasks: record skip, no seed/agent. if surface_sets.get("skip"): reason = surface_sets["skip"] - row["skipped"] = reason - row["verify_note"] = reason + row.skipped = reason + row.verify_note = reason print(f" {task['id']} rep={rep} SKIPPED: {reason}") else: task_needs = set(task.get("needs") or set()) @@ -519,14 +500,14 @@ async def _run_tasks() -> None: try: seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) except TaskSkipped as skip: - row["skipped"] = skip.reason - row["verify_note"] = skip.reason + row.skipped = skip.reason + row.verify_note = skip.reason print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") except Exception as exc: - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "infra_seed" - row["verify_note"] = "" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" print( f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", file=sys.stderr, @@ -539,11 +520,11 @@ async def _run_tasks() -> None: else: if "bug_type" in task_needs and not ctx.get("bug_type"): reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" - row["skipped"] = reason - row["verify_note"] = reason + row.skipped = reason + row.verify_note = reason print(f" {task['id']} rep={rep} SKIPPED: {reason}") else: - agent: dict[str, Any] | None = None + agent: TaskResult | None = None # Agent wrap: API failures and CLI failures are infrastructure. # Contained CLI stops (timeout / error subtypes) return AgentRun. try: @@ -560,10 +541,10 @@ async def _run_tasks() -> None: ) except PromptBindError as exc: # Empty/missing seed IDs in the prompt — not an agent failure. - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "infra_seed" - row["verify_note"] = "" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" print( f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", file=sys.stderr, @@ -576,10 +557,10 @@ async def _run_tasks() -> None: agent_err_class = "infra_api" else: agent_err_class = "infra_cli" - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = agent_err_class - row["verify_note"] = "" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = agent_err_class + row.verify_note = "" print( f" {task['id']} rep={rep} ERROR[{agent_err_class}]: {exc}", file=sys.stderr, @@ -587,74 +568,73 @@ async def _run_tasks() -> None: agent = None if agent is not None: - row.update(agent) + row.apply_agent_result(agent) # Driver-level requested_model is the resolved ID. # Restore run-level intent and retain both identities. - row["requested_model"] = model_alias - row["requested_tier"] = requested_tier - row["resolved_model"] = model_id + row.requested_model = model_alias + row.requested_tier = requested_tier + row.resolved_model = model_id if external: # Empty overlay sets would classify every call # out-of-set; null the counters instead. - row["alternate_calls"] = None - row["out_of_set_calls"] = None + row.alternate_calls = None + row.out_of_set_calls = None # CLI infra stops: timeout + error subtypes except error_max_turns. - stop_reason = agent.get("stop_reason") + stop_reason = agent.stop_reason if driver_name.endswith("-cli") and is_infra_cli_stop_reason( str(stop_reason) if stop_reason is not None else None ): - row["success"] = False - row["error_class"] = "infra_cli" + row.success = False + row.error_class = "infra_cli" if stop_reason == "timeout": - row["error"] = _timeout_error_message(agent) + row.error = _timeout_error_message(agent) else: - notes = [ - n for n in (agent.get("driver_notes") or []) if isinstance(n, str) - ] + notes = [n for n in agent.driver_notes if isinstance(n, str)] detail = "; ".join(notes) if notes else str(stop_reason) - row["error"] = detail - row["verify_note"] = "" + row.error = detail + row.verify_note = "" print( - f" {task['id']} rep={rep} ERROR[infra_cli]: {row['error']}", + f" {task['id']} rep={rep} ERROR[infra_cli]: {row.error}", file=sys.stderr, ) else: verify = task["verify"] try: + agent_row = agent.to_row() ok, note = await verify( plane, ctx, { - "final_text": agent["final_text"], - "calls": agent["calls"], + "final_text": agent.final_text, + "calls": agent_row["calls"], }, ) - row["success"] = bool(ok) - row["verify_note"] = note + row.success = bool(ok) + row.verify_note = note print( f" {task['id']} rep={rep} success={ok} " - f"calls={agent['num_calls']} note={note!r}" + f"calls={agent.num_calls} note={note!r}" ) except TaskSkipped as skip: - row["skipped"] = skip.reason - row["verify_note"] = skip.reason + row.skipped = skip.reason + row.verify_note = skip.reason print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") except Exception as exc: - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "task" - row["verify_note"] = "" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" print( f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr, ) except Exception as exc: # Anything outside seed/driver/verify wraps. - row["success"] = False - row["error"] = f"{type(exc).__name__}: {exc}" - row["error_class"] = "task" - row["verify_note"] = "" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" print(f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr) if ctx.get("project_name"): print( @@ -669,7 +649,7 @@ async def _run_tasks() -> None: if ctx.get("project_name"): print(f" orphaned project: {ctx['project_name']}", file=sys.stderr) - fh.write(json.dumps(row, default=str) + "\n") + fh.write(json.dumps(row.to_row(), default=str) + "\n") fh.flush() await _run_tasks() diff --git a/tests/fixtures/evals_historical_rows.jsonl b/tests/fixtures/evals_historical_rows.jsonl new file mode 100644 index 00000000..51a88478 --- /dev/null +++ b/tests/fixtures/evals_historical_rows.jsonl @@ -0,0 +1,2 @@ +{"run_id": "7a637f5a54664d3eb2fff9ac5a53fb43", "ts": "2026-08-12T17:59:05.498671+00:00", "git_sha": "5da71142cab2d9fd7e8f95be8192ccb17ac3d826", "battery": "6647676edc9e", "surface": "manish-v2", "driver": "codex-cli", "classification": "external", "model": "gpt-5.6-sol", "task_id": "L3", "author": "post-hoc-debias", "rep": 0, "success": true, "verify_note": "release tag 'eval-rc1' present", "skipped": null, "error": null, "error_class": null, "stop_reason": "end_turn", "hit_max_iterations": false, "calls": [{"tool": "release_tag", "class": "out_of_set", "args_chars": 43, "result_tokens": null, "result_chars": 1016, "result_kind": "text", "is_error": false, "duration_ms": 91, "action": "create", "result_tokens_skipped": "no API key / CLI driver has no count_tokens"}], "num_calls": 1, "errored_calls": 0, "alternate_calls": null, "out_of_set_calls": null, "total_result_tokens": 0, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 25.381, "client_tool_calls": [{"tool": "release_tag", "args_chars": 43, "raw_tool": "release_tag"}], "client_tool_call_count": 1, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_pair_mismatch": false, "token_count_failures": 0, "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff720-bf7b-75d2-9b23-eb0b635be673", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"], "result_tokens_skipped_reason": "CLI driver: count_tokens requires Anthropic API key; skipped", "usage": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 263426, "source": "codex_token_count"}} +{"run_id": "625464c995c646429f7cfbcb1a9f5166", "ts": "2026-08-13T03:37:23.554029+00:00", "git_sha": "adf653458ed5788e58acc5e2e9751143df942a5d", "battery": "6425dcc64404", "surface": "full", "driver": "codex-cli", "provider": null, "classification": "exact", "model": "gpt-5.6-sol", "requested_model": "gpt-5.6-sol", "task_id": "R2", "author": "claude", "rep": 0, "success": true, "verify_note": "final text names count 4", "skipped": null, "error": null, "error_class": null, "final_text": "I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4", "stop_reason": "end_turn", "hit_max_iterations": false, "result_pair_mismatch": false, "token_count_failures": 0, "result_tokens_estimated": true, "calls": [{"tool": "list_projects", "class": "alternate", "args_chars": 18, "result_tokens": 315, "result_chars": 1258, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 159}, {"tool": "count_work_items", "class": "optimal", "args_chars": 118, "result_tokens": 64, "result_chars": 253, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 107}], "num_calls": 2, "errored_calls": 0, "alternate_calls": 1, "out_of_set_calls": 0, "total_result_tokens": 379, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 29.679, "client_tool_calls": [{"tool": "list_projects", "args_chars": 18, "raw_tool": "list_projects"}, {"tool": "count_work_items", "args_chars": 118, "raw_tool": "count_work_items"}], "client_tool_call_count": 2, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_tokens_mode": "estimated", "result_token_count_method": "chars_div_4", "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff932-5598-7d62-9ab9-30c6bf5fca15", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"], "usage": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 292690, "source": "codex_token_count"}} diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py index a7194c70..1917b7bb 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/test_evals_api_driver.py @@ -159,7 +159,7 @@ async def session_factory(_params): assert run.model == "dummy-model-actual" assert run.stopped_reason == "end_turn" assert run.provider_stop_reason == "dummy_complete" - assert run.usage_per_iteration == [{"in": 7, "out": 2, "cache_read": 0, "cache_write": 0}] + assert run.usage_per_iteration == [Usage(7, 2, 0, 0)] def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): @@ -203,11 +203,7 @@ def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): assert run.final_text == "done" assert run.stopped_reason == "end_turn" assert run.cum_input_tokens == 60 - assert run.usage_per_iteration == [ - {"in": 10, "out": 2, "cache_read": 3, "cache_write": 1}, - {"in": 20, "out": 4, "cache_read": 6, "cache_write": 0}, - {"in": 30, "out": 6, "cache_read": 9, "cache_write": 0}, - ] + assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] assert [call["result_tokens"] for call in run.calls] == [ estimate_result_tokens(len("first result")), diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 215a5f17..55d6887d 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -18,6 +18,7 @@ from evals import seed as seed_mod from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize +from evals.results import RESULT_SCHEMA_VERSION from evals.run import ( is_infra_cli_stop_reason, load_resume_skip_keys, @@ -242,6 +243,7 @@ def boom_seed(plane, run_id, needs, ctx): rows = _data_rows(out) assert len(rows) == 1 row = rows[0] + assert row["schema_version"] == RESULT_SCHEMA_VERSION assert row["error_class"] == "infra_seed" assert row["success"] is False assert "HttpError" in (row["error"] or "") @@ -252,6 +254,7 @@ def boom_seed(plane, run_id, needs, ctx): assert row["resolved_model"] == "sonnet" assert row["model"] == "sonnet" meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert meta["schema_version"] == RESULT_SCHEMA_VERSION assert meta["requested_tier"] == "standard" assert meta["resolved_model"] == "sonnet" @@ -785,8 +788,8 @@ def test_load_rows_dedupe_latest_wins(tmp_path: Path): p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") loaded = load_rows(p) # default dedupe=latest assert len(loaded) == 1 - assert loaded[0]["num_calls"] == 9 - assert loaded[0]["success"] is False + assert loaded[0].num_calls == 9 + assert loaded[0].success is False def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index daa7e3b8..4c927269 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -29,6 +29,7 @@ write_antigravity_mcp_config, write_opencode_mcp_config, ) +from evals.drivers.cli import CliDriver, CliLaunch, CliOutput from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -577,6 +578,100 @@ def test_agent_run_to_harness_propagates_proxy_fields(): # --------------------------------------------------------------------------- +def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): + clock = {"now": 0.0} + monkeypatch.setattr("evals.drivers.cli.time.perf_counter", lambda: clock["now"]) + + class MinimalCliDriver(CliDriver): + name = "minimal-cli" + temp_dir_prefix = "plane-eval-minimal-" + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir, child_env + # Harness-owned setup takes five seconds on the fake clock. The + # persisted wall time must start after this hook returns. + clock["now"] = 5.0 + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del model, max_turns, system, launch + return ["minimal", prompt] + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del proc, task_cwd, max_turns, notes + return CliOutput( + final_text="done", + calls=[ + {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, + {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, + ], + ) + + def write_complete_sidecar(path: Path, tool: str) -> None: + rows = [ + { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + }, + {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + success_driver: MinimalCliDriver + + def success_runner(cmd, **kwargs): + write_complete_sidecar(success_driver.sidecar_path, "proxy_first") + clock["now"] = 7.0 + return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") + + success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) + success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert success.call_source == "proxy" + assert [call["tool"] for call in success.calls] == ["proxy_first"] + assert success.wall_time_s == 2.0 + + timeout_driver: MinimalCliDriver + + def timeout_runner(cmd, **kwargs): + write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") + clock["now"] = 8.0 + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) + + timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) + timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert timed_out.stopped_reason == "timeout" + assert timed_out.call_source == "proxy" + assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] + assert timed_out.wall_time_s == 3.0 + + def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path: Path): seen: dict = {} diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index 1247fc6a..5cb68830 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -26,6 +26,7 @@ summarize, wilson_interval, ) +from evals.results import RESULT_SCHEMA_VERSION, CallRecord, TaskResult, Usage from evals.run import ( is_meta_or_non_task_row, load_resume_skip_keys, @@ -107,7 +108,58 @@ def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): p.write_text("\n".join(lines) + "\n", encoding="utf-8") rows = load_rows(p) assert len(rows) == 1 - assert rows[0]["task_id"] == "R1" + assert rows[0].task_id == "R1" + + +def test_task_result_schema_round_trip_owns_usage_shape(): + result = TaskResult( + task_id="R1", + calls=[ + CallRecord( + tool="find_work_items", + classification="optimal", + result_tokens=3, + result_tokens_estimated=False, + result_token_count_method="backend", + ) + ], + num_calls=1, + usage_per_iteration=[Usage(10, 2, 3, 4)], + ) + + row = result.to_row() + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] + loaded = TaskResult.from_row(row) + assert loaded.calls[0].tool == "find_work_items" + assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] + + +def test_real_historical_rows_parse_and_report_with_backward_defaults(): + fixture = Path(__file__).parent / "fixtures" / "evals_historical_rows.jsonl" + rows = load_rows(fixture) + + assert [row.schema_version for row in rows] == [0, 0] + by_task = {row.task_id: row for row in rows} + battery4 = by_task["L3"] + assert battery4.final_text == "" + assert battery4.result_tokens_estimated is None + assert battery4.alternate_calls is None + assert battery4.calls[0].result_tokens is None + assert battery4.calls[0].action == "create" + + battery5 = by_task["R2"] + assert battery5.final_text.endswith("\n4") + assert battery5.result_tokens_estimated is True + assert [call.result_tokens for call in battery5.calls] == [315, 64] + + summary = summarize(rows) + assert summary["L3"]["success"] == "1/1" + assert summary["L3"]["med_calls"] == 1 + assert summary["L3"]["result_tokens_mode"] == "unavailable" + assert summary["R2"]["success"] == "1/1" + assert summary["R2"]["med_calls"] == 2 + assert summary["R2"]["result_tokens_mode"] == "estimated" def test_dedupe_rows_latest_pure(): @@ -118,9 +170,9 @@ def test_dedupe_rows_latest_pure(): ] out = dedupe_rows_latest(rows) assert len(out) == 2 - by_id = {r["task_id"]: r for r in out} - assert by_id["R1"]["num_calls"] == 5 - assert by_id["R2"]["num_calls"] == 3 + by_id = {r.task_id: r for r in out} + assert by_id["R1"].num_calls == 5 + assert by_id["R2"].num_calls == 3 def test_summarize_aggregate_wilson_and_call_variance(): From de9190c898e16de056bf1e8fd9e6b5487a125117 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 11:59:22 +0530 Subject: [PATCH 11/93] Grade answers by contract, not by prose style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifiers that scan free text measure how an agent writes, not what the tool surface can do. C2 had run 25 times and passed 6 — split by agent that is claude-cli 6 of 8 and codex-cli 0 of 17, because it hunted for changelog content in prose and a terse-but-correct answer never matched. Read at face value that says a surface is worse under GPT, which is false. R2 had the same shape: the prompt asked for "the integer count only" while the verifier matched the number anywhere in the text, so "There are four urgent open work items" failed. Ten tasks now state an explicit output contract in the prompt and match it exactly — the pattern the de-bias tasks already used. A correct answer in any style passes; an absent or wrong one still fails, which the canary proves by rejecting a do-nothing agent on every task. Prompts are part of the catalog, so the battery fingerprint moves from 6425dcc64404 to 182c4748ef14 and results for changed tasks are no longer comparable with battery3/4/5. That is the honest consequence of fixing the questions rather than the answers, so the README says it and report warns when a table spans fingerprints instead of silently comparing different questions. Five unrelated semantic gaps were found and deliberately left alone rather than tuned: R7 accepts any exact state without proving the transition is legal, W8 cannot verify "yesterday" because the worklog surface has no logged-date field, W10 checks a page's name but not its body, W3 checks one distinctive phrase of the requested comment, and L2 counts activities without checking the phrases. Co-Authored-By: Claude Fable 5 --- evals/README.md | 12 ++- evals/report.py | 21 ++++ evals/tasks/__init__.py | 6 ++ evals/tasks/common.py | 35 +++++++ evals/tasks/cross.py | 52 ++++++--- evals/tasks/debias.py | 123 +++++++--------------- evals/tasks/read.py | 151 ++++++++++++++------------- tests/test_evals_debias_verifiers.py | 78 +++++++++++--- tests/test_evals_output_contracts.py | 119 +++++++++++++++++++++ tests/test_evals_report_ops.py | 20 ++++ tests/test_evals_verifiers.py | 19 +++- 11 files changed, 443 insertions(+), 193 deletions(-) create mode 100644 tests/test_evals_output_contracts.py diff --git a/evals/README.md b/evals/README.md index 99506991..6ffacb79 100644 --- a/evals/README.md +++ b/evals/README.md @@ -144,6 +144,12 @@ denominators, as are rows with recorded errors. Result-token columns use `~` for `*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not recorded. +Every result row carries a `battery` fingerprint derived from the selected catalog's prompts +and tool metadata. Compare rows only when their fingerprints match: a table that mixes +fingerprints is comparing different questions, even when task IDs are the same. In particular, +results from a task whose output contract changed are not directly comparable with its rows in +older batteries. `evals.report --table` warns when its input rows span fingerprints. + ## Running surfaces in parallel Tasks that touch **workspace-scoped** fixtures (release tags, customer properties) collide @@ -208,8 +214,10 @@ preserve the assembly order in `tasks/__init__.py`. Mutation verifiers must read the resulting state through the Plane API. Read verifiers must derive the expected facts from the API or seed context and match an explicit answer contract -or exact seeded values. For numeric answers, prefer a prompt such as `Answer with a line -'count: N'` and the shared contract matcher; a loose substring can make `4` match `24`. +instead of scanning free-form prose. Use exact `field: value` lines and the shared contract +matchers; for numeric answers, prefer a prompt such as `Answer with a line 'count: N'`. A +loose substring can make `4` match `24`, and prose matching can accidentally grade an agent's +writing habits instead of its answer. **Check the shape the API actually returns.** Dates come back as timestamps (`2026-08-12T00:00:00Z`), so comparing one to a bare `2026-08-12` silently never matches — diff --git a/evals/report.py b/evals/report.py index fe11e477..3784b128 100644 --- a/evals/report.py +++ b/evals/report.py @@ -628,6 +628,26 @@ def _surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: return path.stem +def warn_if_table_mixes_batteries(file_rows: list[tuple[str, list[ResultRow]]]) -> bool: + """Warn when table columns contain rows from different task batteries.""" + by_label: dict[str, set[str]] = {} + all_fingerprints: set[str] = set() + for label, rows in file_rows: + fingerprints = {_task_result(row).battery or "" for row in rows if not is_meta_row(row)} + if fingerprints: + by_label[label] = fingerprints + all_fingerprints.update(fingerprints) + if len(all_fingerprints) <= 1: + return False + detail = "; ".join(f"{label}={','.join(sorted(values))}" for label, values in by_label.items()) + print( + "warning: table spans battery fingerprints; these rows were graded on " + f"different task prompts/questions and are not directly comparable ({detail})", + file=sys.stderr, + ) + return True + + def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description="Summarize eval JSONL results") p.add_argument( @@ -680,6 +700,7 @@ def main(argv: list[str] | None = None) -> int: n += 1 used_labels.add(label) labeled.append((label, rows)) + warn_if_table_mixes_batteries(labeled) table = build_multi_surface_table(labeled) sys.stdout.write(render_multi_surface_table(table, markdown=args.markdown)) return 0 diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index 1394b355..472e454d 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -10,6 +10,7 @@ PromptBindError, TaskSkipped, as_id, + contract_values, count_open_urgent, find_item_by_name, find_items_by_name, @@ -18,6 +19,8 @@ ids, is_not_found, reports_contract_int, + reports_contract_value, + reports_contract_values, reports_exact_int, state_group, state_name, @@ -266,6 +269,7 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "TaskSkipped", "as_id", "battery_fingerprint", + "contract_values", "count_open_urgent", "find_item_by_name", "find_items_by_name", @@ -275,6 +279,8 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "ids", "is_not_found", "reports_contract_int", + "reports_contract_value", + "reports_contract_values", "reports_exact_int", "resolve_surface_tool_sets", "state_group", diff --git a/evals/tasks/common.py b/evals/tasks/common.py index 2b17b858..e52edf49 100644 --- a/evals/tasks/common.py +++ b/evals/tasks/common.py @@ -4,6 +4,7 @@ import re import string +from collections import Counter from typing import Any from plane.errors.errors import HttpError @@ -135,6 +136,37 @@ def reports_contract_int(text: str, truth: int) -> bool: return False +def contract_values(text: str, field: str) -> list[str]: + """Return non-empty values from exact ``field: value`` contract lines. + + The field name is case-insensitive, as with :func:`reports_contract_int`, + while the value is preserved for exact comparison. Prose, bullets, inline + mentions, and malformed/empty contract lines are ignored. + """ + values: list[str] = [] + pattern = re.compile(rf"\s*{re.escape(field)}:\s*(.*?)\s*", flags=re.IGNORECASE) + for line in (text or "").splitlines(): + match = pattern.fullmatch(line) + if match and match.group(1): + values.append(match.group(1)) + return values + + +def reports_contract_value(text: str, field: str, truth: str) -> bool: + """True when exactly one ``field: value`` line equals ``truth`` exactly.""" + return contract_values(text, field) == [str(truth)] + + +def reports_contract_values(text: str, field: str, truths: list[str] | tuple[str, ...]) -> bool: + """True when contract lines equal the expected value multiset. + + Ordering is deliberately ignored: the output contract defines one exact + fact per line, not a presentation order. Missing, duplicate, or extra field + lines fail. + """ + return Counter(contract_values(text, field)) == Counter(str(value) for value in truths) + + def as_id(obj: Any) -> str | None: if obj is None: return None @@ -260,6 +292,9 @@ def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: "reports_exact_int", "whole_answer_int", "reports_contract_int", + "contract_values", + "reports_contract_value", + "reports_contract_values", "as_id", "ids", "find_items_by_name", diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py index 6bd7ee0b..b64699ca 100644 --- a/evals/tasks/cross.py +++ b/evals/tasks/cross.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Any from evals.seed import ( @@ -11,7 +12,15 @@ RELEASE_CHANGELOG_TEXT, RELEASE_NAME, ) -from evals.tasks.common import as_id, find_item_by_name, get_final_text, ids, word_boundary +from evals.tasks.common import ( + as_id, + contract_values, + find_item_by_name, + get_final_text, + ids, + reports_contract_value, + reports_contract_values, +) async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: @@ -130,35 +139,44 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """C2: final text mentions release 1.2.0 and at least one seeded changelog phrase.""" + """C2: exact contract fields report the release and every changelog item.""" final_text = get_final_text(run) notes: list[str] = [] ok = True - if not word_boundary(RELEASE_NAME).search(final_text): + if not reports_contract_value(final_text, "release", RELEASE_NAME): ok = False - notes.append(f"missing release name {RELEASE_NAME!r}") + notes.append(f"release values={contract_values(final_text, 'release')!r}; want [{RELEASE_NAME!r}]") else: - notes.append(f"names {RELEASE_NAME}") + notes.append(f"release={RELEASE_NAME!r}") + changelog = ctx.get("release_changelog_text") or RELEASE_CHANGELOG_TEXT - # Match distinctive fragments from the seeded changelog. - fragments = ["OAuth login hardening", "webhook retry backoff"] - hit = [f for f in fragments if word_boundary(f).search(final_text)] - if not hit: - # Also accept substring of full changelog without word-boundary if short. - if changelog[:40].casefold() not in final_text.casefold(): - ok = False - notes.append("missing changelog content") - else: - notes.append("changelog substring present") + markers = list(re.finditer(r"Changelog entry\s+[^:]+:\s*", changelog, flags=re.IGNORECASE)) + shipped: list[str] = [] + for index, marker in enumerate(markers): + end = markers[index + 1].start() if index + 1 < len(markers) else len(changelog) + item = changelog[marker.end() : end].strip().rstrip(".").strip() + if item: + shipped.append(item) + if not shipped: + return False, f"seeded changelog does not contain parseable entries: {changelog!r}" + if not reports_contract_values(final_text, "shipped", shipped): + ok = False + notes.append(f"shipped values={contract_values(final_text, 'shipped')!r}; want {shipped!r}") else: - notes.append(f"changelog phrases {hit}") + notes.append(f"{len(shipped)} exact shipped items") return ok, "; ".join(notes) C2_TASK: dict[str, Any] = { "id": "C2", "tags": {"read", "tier1"}, - "prompt": (f"What shipped in release {RELEASE_NAME}? Summarize the changelog."), + "prompt": ( + f"What shipped in release {RELEASE_NAME}? Summarize the changelog in any prose " + f"you like, then provide these exact contract lines: 'release: {RELEASE_NAME}' " + "and one 'shipped: ' line per changelog item. " + "For each 'shipped:' value, copy only the text after the changelog entry label, " + "without its sentence-ending punctuation." + ), "optimal_calls": 2, "optimal_tools": {"list_releases", "get_release_changelog"}, "alternate_tools": { diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 2211e17e..93c33519 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from typing import Any from plane.models.query_params import RetrieveQueryParams @@ -18,7 +17,15 @@ W3_TITLE, W8_TITLE, ) -from evals.tasks.common import get_final_text, ids, reports_contract_int, state_name, word_boundary +from evals.tasks.common import ( + contract_values, + get_final_text, + ids, + reports_contract_int, + reports_contract_value, + reports_contract_values, + state_name, +) I1_TITLE = R1_TITLE @@ -119,7 +126,7 @@ async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I2: final text names the state of the identifier-target item.""" + """I2: final text reports the identifier-target state via ``state: NAME``.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] wid = (ctx.get("items") or {}).get(I2_TITLE) @@ -130,9 +137,9 @@ async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if not name: return False, "target state name unresolved" final_text = get_final_text(run) - if word_boundary(name).search(final_text): - return True, f"final text names state {name!r}" - return False, f"final text missing state {name!r}" + if reports_contract_value(final_text, "state", name): + return True, f"final text reports state {name!r} via contract" + return False, f"state values={contract_values(final_text, 'state')!r}; want [{name!r}]" I2_TASK: dict[str, Any] = { @@ -141,7 +148,7 @@ async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"read", "tier1", "id_in_hand", "debias"}, "prompt": ( "In project {project}, what is the current state of work item " - "{work_item_identifier}? Answer with the state name only." + "{work_item_identifier}? Return exactly one line: 'state: '." ), "prompt_bind": _bind_item_identifier(I2_TITLE), "optimal_calls": 1, @@ -305,74 +312,8 @@ async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup } -def _l1_duration_reported(final_text: str) -> bool: - """Numeric duration only: whole-word 90 or 1.5 (not English 'ninety').""" - return bool(word_boundary("90").search(final_text)) or bool(re.search(r"\b1\.5\b", final_text)) - - -def _l1_person_names_from_summary(sum_rows: Any) -> list[str]: - """Best-effort actor/assignee display strings from project worklog summary rows.""" - names: list[str] = [] - for row in sum_rows or []: - dump = row.model_dump() if hasattr(row, "model_dump") else {} - if not isinstance(dump, dict): - dump = {} - candidates: list[Any] = [] - for attr in ( - "actor", - "user", - "display_name", - "owned_by", - "created_by", - "assignee", - "email", - "first_name", - "last_name", - ): - v = getattr(row, attr, None) - if v is None and dump: - v = dump.get(attr) - if v is None: - continue - if hasattr(v, "display_name") or hasattr(v, "email"): - candidates.append( - getattr(v, "display_name", None) or getattr(v, "email", None) or getattr(v, "id", None) - ) - elif isinstance(v, dict): - candidates.append(v.get("display_name") or v.get("email") or v.get("id")) - else: - candidates.append(v) - for c in candidates: - s = str(c or "").strip() - if s and s not in names: - names.append(s) - return names - - -def _l1_summary_substance(final_text: str, *, title: str, sum_rows: Any) -> bool: - """Summary half of L1: item title, person from summary, or words summary/total. - - Deliberately does *not* accept bare 'logged' / 'worklog' — the prompt asks to - report the project worklog summary (who/what has time logged). - """ - low = final_text.casefold() - if "summary" in low or "total" in low: - return True - if title and word_boundary(title).search(final_text): - return True - for person in _l1_person_names_from_summary(sum_rows): - if len(person) >= 2 and word_boundary(person).search(final_text): - return True - return False - - async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L1: 90-minute work log on the correct item AND final text reports duration + summary. - - Duration: numeric whole-word ``90`` or ``1.5`` only (not English 'ninety'). - Summary substance: item title, a person/assignee from the project summary, or - the words ``summary`` / ``total``. Bare "90 minutes of work" fails by design. - """ + """L1: 90-minute log exists and exact contract lines report the API summary.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] wid = (ctx.get("items") or {}).get(L1_TITLE) @@ -385,24 +326,33 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if 90 not in durations: return False, f"no 90-minute work log on target item {wid}; durations={durations}" - sum_rows: list[Any] = [] try: summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) sum_rows = list(raw or []) - except Exception: - # Summary fetch is optional for person names; duration + title/summary/total still work. - sum_rows = [] + except Exception as exc: + return False, f"project worklog summary failed: {exc}" + + summary_ids: list[str] = [] + for row in sum_rows: + dump = row.model_dump() if hasattr(row, "model_dump") else (row if isinstance(row, dict) else {}) + value = getattr(row, "issue_id", None) or getattr(row, "work_item_id", None) + if value is None and isinstance(dump, dict): + value = dump.get("issue_id") or dump.get("work_item_id") + item_id = str(value or "").strip() + if item_id and item_id not in summary_ids: + summary_ids.append(item_id) + if str(wid) not in summary_ids: + return False, f"target item {wid} missing from project worklog summary ids={summary_ids!r}" final_text = get_final_text(run) - if not _l1_duration_reported(final_text): - return False, "final text missing logged duration (numeric 90 or 1.5)" - if not _l1_summary_substance(final_text, title=L1_TITLE, sum_rows=sum_rows): + if not reports_contract_value(final_text, "logged-minutes", "90"): + return False, f"logged-minutes values={contract_values(final_text, 'logged-minutes')!r}; want ['90']" + if not reports_contract_values(final_text, "summary-work-item-id", summary_ids): return False, ( - "final text lacks worklog summary substance " - "(need item title, person from summary, or words 'summary'/'total')" + f"summary-work-item-id values={contract_values(final_text, 'summary-work-item-id')!r}; want {summary_ids!r}" ) - return True, f"90m log on {wid} + final text reports duration and summary substance" + return True, f"90m log on {wid} + exact contract for {len(summary_ids)} summary row(s)" L1_TASK: dict[str, Any] = { @@ -411,7 +361,10 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"write", "read", "tier1", "long_tail", "debias"}, "prompt": ( f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " - f"'{L1_TITLE}', then report the project's worklog summary (who/what has time logged)." + f"'{L1_TITLE}', then report the project's worklog summary. End with exactly " + "one 'logged-minutes: 90' line and one " + "'summary-work-item-id: ' line for every row returned " + "by the project worklog summary. Include no other lines with those prefixes." ), "optimal_calls": 3, "optimal_tools": {"list_work_items", "create_work_log", "get_project_worklog_summary"}, diff --git a/evals/tasks/read.py b/evals/tasks/read.py index 656d1a2c..cb12ab1b 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -6,21 +6,19 @@ from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, R5_TITLE from evals.tasks.common import ( + contract_values, count_open_urgent, find_item_by_name, get_final_text, + reports_contract_int, + reports_contract_value, + reports_contract_values, state_name, - word_boundary, ) async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R1: final text must name the target item's state and no other seeded state. - - Matching rule: word-boundary, case-insensitive regex on the exact state name - resolved from the API at verify time (never hardcoded). Additionally fail if - any *other* project state name also matches (blocks guessing/list_states echo). - """ + """R1: final text must report the API-resolved state via ``state: NAME``.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] title = R1_TITLE @@ -36,17 +34,9 @@ async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup return False, "could not resolve expected state name from API" final_text = get_final_text(run) - if not word_boundary(expected).search(final_text): - return False, f"final text missing state name {expected!r}" - - other_states = [n for n in (ctx.get("state_names") or []) if n and n.casefold() != expected.casefold()] - collisions = [n for n in other_states if word_boundary(n).search(final_text)] - if collisions: - return ( - False, - f"final text names other state(s) {collisions!r} besides expected {expected!r}", - ) - return True, f"final text names only state {expected!r}" + if not reports_contract_value(final_text, "state", expected): + return False, f"final text must contain exactly 'state: {expected}'" + return True, f"final text reports state {expected!r} via contract" R1_TASK: dict[str, Any] = { @@ -54,7 +44,7 @@ async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"read", "tier1"}, "prompt": ( "In project {project}, what is the current state of the work item titled " - f"'{R1_TITLE}'? Answer with the state name." + f"'{R1_TITLE}'? Return exactly one line: 'state: '." ), "optimal_calls": 1, "optimal_tools": {"list_work_items"}, @@ -86,21 +76,23 @@ async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R2: final text must contain the exact urgent-open count (word-boundary).""" + """R2: final text reports the urgent-open count via ``count: N``.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] expected = count_open_urgent(plane, workspace_slug, project_id) final_text = get_final_text(run) - # Word-boundary on the decimal form of the count (blocks "4" matching "24"). - if not word_boundary(str(expected)).search(final_text): - return False, f"final text missing urgent-open count {expected}" - return True, f"final text names count {expected}" + if not reports_contract_int(final_text, expected): + return False, f"final text missing contract count: {expected} (need 'count: {expected}')" + return True, f"final text reports urgent-open count {expected} via contract" R2_TASK: dict[str, Any] = { "id": "R2", "tags": {"read", "tier1"}, - "prompt": ("In project {project}, how many urgent open work items are there? Answer with the integer count only."), + "prompt": ( + "In project {project}, how many urgent open work items are there? " + "Return exactly one line of the form 'count: N', where N is the integer count." + ), "optimal_calls": 1, "optimal_tools": {"count_work_items"}, "alternate_tools": { @@ -130,22 +122,22 @@ async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R3: final text must include each seeded assigned-to-me / due-this-week title.""" + """R3: ``item: TITLE`` lines exactly match the seeded due-title set.""" titles = list(ctx.get("r3_due_titles") or []) if not titles: return False, "no R3 due titles in seed ctx" final_text = get_final_text(run) - missing = [t for t in titles if not word_boundary(t).search(final_text)] - if missing: - return False, f"final text missing title(s) {missing!r}" - return True, f"final text names {len(titles)} due-this-week assigned items" + if not reports_contract_values(final_text, "item", titles): + return False, f"item contract values={contract_values(final_text, 'item')!r}; want {titles!r}" + return True, f"final text reports exactly {len(titles)} due-this-week assigned items" R3_TASK: dict[str, Any] = { "id": "R3", "tags": {"read", "tier1"}, "prompt": ( - "In project {project}, list work items assigned to me that are due this week. Answer with their titles." + "In project {project}, list work items assigned to me that are due this week. " + "Return one line per result as 'item: ' and no other 'item:' lines." ), "optimal_calls": 2, "optimal_tools": {"get_me", "list_work_items"}, @@ -175,26 +167,34 @@ async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R4: final text must mention the active cycle name and the overdue item title.""" + """R4: contract reports the active cycle, all its items, and overdue items.""" final_text = get_final_text(run) notes: list[str] = [] ok = True - if not word_boundary(CYCLE_CURRENT).search(final_text): + if not reports_contract_value(final_text, "cycle", CYCLE_CURRENT): + ok = False + notes.append(f"cycle values={contract_values(final_text, 'cycle')!r}; want [{CYCLE_CURRENT!r}]") + else: + notes.append(f"cycle={CYCLE_CURRENT!r}") + + active_ids = {str(value) for value in (ctx.get("r4_active_item_ids") or [])} + active_titles = [str(title) for title, item_id in (ctx.get("items") or {}).items() if str(item_id) in active_ids] + if not active_titles: + ok = False + notes.append("no active-cycle titles in seed ctx") + elif not reports_contract_values(final_text, "item", active_titles): + ok = False + notes.append(f"item values={contract_values(final_text, 'item')!r}; want {active_titles!r}") + else: + notes.append(f"{len(active_titles)} active-cycle items") + + overdue = str(ctx.get("r4_overdue_title") or "") + expected_overdue = [overdue] if overdue else ["none"] + if not reports_contract_values(final_text, "overdue", expected_overdue): ok = False - notes.append(f"missing active cycle {CYCLE_CURRENT!r}") + notes.append(f"overdue values={contract_values(final_text, 'overdue')!r}; want {expected_overdue!r}") else: - notes.append(f"names {CYCLE_CURRENT}") - overdue = ctx.get("r4_overdue_title") - if overdue: - if not word_boundary(overdue).search(final_text): - # Soft: also accept "overdue" keyword + any active item title. - if "overdue" not in final_text.casefold(): - ok = False - notes.append(f"missing overdue title {overdue!r}") - else: - notes.append("mentions overdue (title not exact)") - else: - notes.append(f"names overdue {overdue!r}") + notes.append(f"overdue={expected_overdue!r}") return ok, "; ".join(notes) @@ -203,7 +203,10 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"read", "tier1"}, "prompt": ( "In project {project}, what is in the active cycle, and is anything overdue? " - f"Name the cycle (expect '{CYCLE_CURRENT}') and any overdue item titles." + f"Use these exact contract lines: one 'cycle: {CYCLE_CURRENT}' line, one " + "'item: ' line for every item in that cycle, and one " + "'overdue: ' line for every overdue item. If none " + "are overdue, use 'overdue: none'." ), "optimal_calls": 2, "optimal_tools": {"list_cycles", "list_work_items"}, @@ -233,13 +236,12 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R5: final text must include seeded comment phrases (word-boundary).""" + """R5: ``comment: TEXT`` lines exactly match the seeded comments.""" phrases = list(ctx.get("r5_comment_phrases") or R5_COMMENT_PHRASES) final_text = get_final_text(run) - missing = [p for p in phrases if not word_boundary(p).search(final_text)] - if missing: - return False, f"final text missing comment phrase(s) {missing!r}" - return True, f"final text names {len(phrases)} discussion phrases" + if not reports_contract_values(final_text, "comment", phrases): + return False, f"comment values={contract_values(final_text, 'comment')!r}; want {phrases!r}" + return True, f"final text reports exactly {len(phrases)} seeded comments" R5_TASK: dict[str, Any] = { @@ -247,7 +249,9 @@ async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"read", "tier1"}, "prompt": ( f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " - "Include the key phrases from its comments." + "You may summarize in prose, but end with one contract line per comment: " + "'comment: '. Copy the comment text exactly and include " + "no other 'comment:' lines." ), "optimal_calls": 2, "optimal_tools": {"list_work_items", "list_work_item_comments"}, @@ -277,20 +281,14 @@ async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R6: final text must name the project that has more open bugs (resolved at verify).""" + """R6: final text reports the winning project via ``project: NAME``.""" expected = ctx.get("r6_more_bugs_project") or ctx.get("second_project_name") if not expected: return False, "second project name missing from seed ctx" final_text = get_final_text(run) - # Match the full project name or the distinctive " B" suffix run8 form. - if word_boundary(expected).search(final_text): - return True, f"final text names project with more bugs {expected!r}" - # Allow matching just the identifier-ish trailing token (e.g. run8 + B). - run8 = ctx.get("run8") or "" - alt = f"EVAL {run8} B" - if word_boundary(alt).search(final_text) or (run8 and run8 in final_text and " B" in final_text): - return True, f"final text names second project ({alt})" - return False, f"final text missing project with more bugs {expected!r}" + if not reports_contract_value(final_text, "project", expected): + return False, f"project values={contract_values(final_text, 'project')!r}; want [{expected!r}]" + return True, f"final text reports project with more bugs {expected!r}" R6_TASK: dict[str, Any] = { @@ -299,7 +297,7 @@ async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "prompt": ( "Across the eval projects created for this run (main project {project} and its " "sibling 'B' project), which project has more open Bug-typed work items? " - "Answer with the project name." + "Return exactly one line: 'project: '." ), "optimal_calls": 3, "optimal_tools": {"list_projects", "list_work_items", "resolve_work_item_type"}, @@ -338,22 +336,26 @@ async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R7 (extra): final text names at least one legal next state for the R1 item. + """R7 (extra): contract names project states or explicitly says unrestricted. - Resolves available completed/started/unstarted states at verify time and - requires a word-boundary hit on one of them (or explicit 'unrestricted'). + This preserves the verifier's existing semantic ceiling: the full surface + exposes project states, not authoritative workflow transition evaluation. + The output match itself is structural and exact. """ workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) names = [(s.name or "").strip() for s in (page.results or []) if (s.name or "").strip()] final_text = get_final_text(run) - if "unrestricted" in final_text.casefold() or "any state" in final_text.casefold(): + reported = contract_values(final_text, "transition") + if reported == ["unrestricted"]: return True, "agent reported unrestricted transitions" - hits = [n for n in names if word_boundary(n).search(final_text)] - if not hits: - return False, f"final text names none of project states {names}" - return True, f"final text names state(s) {hits}" + if not reported: + return False, "final text has no 'transition: ' contract lines" + unknown = [value for value in reported if value not in names] + if unknown: + return False, f"transition values {unknown!r} are not exact project state names; have {names!r}" + return True, f"final text reports project state(s) {reported!r} via contract" R7_TASK: dict[str, Any] = { @@ -361,8 +363,9 @@ async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"read", "tier1", "extra"}, "prompt": ( f"In project {{project}}, what states can the work item '{R1_TITLE}' " - "legally transition to under workflow rules? List the state names " - "(or say unrestricted if none)." + "legally transition to under workflow rules? Return one line per state as " + "'transition: '. If transitions are unrestricted, return " + "exactly 'transition: unrestricted'." ), # Extra: exercises list_available_transitions. "optimal_calls": 2, diff --git a/tests/test_evals_debias_verifiers.py b/tests/test_evals_debias_verifiers.py index 623a1fe9..98238bb7 100644 --- a/tests/test_evals_debias_verifiers.py +++ b/tests/test_evals_debias_verifiers.py @@ -174,6 +174,20 @@ async def _go(): return asyncio.run(_go()) +def test_i2_exact_state_contract_passes(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[st], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("state: Backlog")) + assert ok is True, note + + return asyncio.run(_go()) + + # --------------------------------------------------------------------------- # I3 — cycle membership by UUIDs # --------------------------------------------------------------------------- @@ -318,7 +332,7 @@ async def _go(): ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} ok, note = await verify_l1(plane, ctx, _run("")) assert ok is False, note - assert "duration" in note.lower() or "summary" in note.lower() + assert "logged-minutes" in note.lower() return asyncio.run(_go()) @@ -340,45 +354,39 @@ async def _go(): return asyncio.run(_go()) -def test_l1_logged_1_5_hours_with_total_passes(): - """Reviewer: numeric 1.5 is a valid duration token (with summary substance).""" +def test_l1_prose_with_correct_facts_but_without_contract_fails(): + """Correct facts in prose do not satisfy the explicit output contract.""" async def _go(): plane = _L1Plane([90], summary_ids=["wi-l1"]) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} ok, note = await verify_l1(plane, ctx, _run("Logged 1.5 hours total.")) - assert ok is True, note + assert ok is False, note return asyncio.run(_go()) def test_l1_ninety_minutes_of_work_fails_by_design(): - """Calibration: '90 minutes of work' FAILS by design. - - The prompt asks to report the project worklog summary (who/what has time - logged). An answer naming neither who nor what (no title, no person, no - 'summary'/'total') has not done that half of the task — even with a correct - numeric duration. - """ + """Calibration: prose without contract lines fails by design.""" async def _go(): plane = _L1Plane([90], summary_ids=["wi-l1"]) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} ok, note = await verify_l1(plane, ctx, _run("90 minutes of work")) assert ok is False, note - assert "summary" in note.lower() + assert "logged-minutes" in note.lower() return asyncio.run(_go()) -def test_l1_90m_log_and_summary_text_passes(): +def test_l1_exact_duration_and_summary_contract_passes(): async def _go(): plane = _L1Plane([90], summary_ids=["wi-l1"]) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} ok, note = await verify_l1( plane, ctx, - _run("Logged 90 minutes. Worklog summary: 1 item with time logged."), + _run("logged-minutes: 90\nsummary-work-item-id: wi-l1"), ) assert ok is True, note @@ -638,6 +646,17 @@ def test_reports_contract_int_unit(): assert reports_contract_int("count: 9\ncount: 3", 9) is False +def test_exact_line_contract_helpers_unit(): + from evals.tasks import contract_values, reports_contract_value, reports_contract_values + + text = "prose mentions state Done\nSTATE: In Progress\nitem: B\nitem: A" + assert contract_values(text, "state") == ["In Progress"] + assert reports_contract_value(text, "state", "In Progress") is True + assert reports_contract_value("- state: In Progress", "state", "In Progress") is False + assert reports_contract_values(text, "item", ["A", "B"]) is True + assert reports_contract_values("item: A\nitem: A", "item", ["A"]) is False + + # --------------------------------------------------------------------------- # Sample of 6 existing verifiers (untouched + wrong-value) # --------------------------------------------------------------------------- @@ -691,6 +710,21 @@ async def _go(): return asyncio.run(_go()) +def test_existing_r1_exact_state_contract_passes(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("state: In Progress")) + assert ok is True, note + + return asyncio.run(_go()) + + class _R2Plane: def __init__(self, count: int): self._count = count @@ -833,6 +867,22 @@ async def _go(): return asyncio.run(_go()) +def test_existing_c2_exact_release_and_shipped_contract_passes(): + async def _go(): + ok, note = await verify_c2( + object(), + { + "release_changelog_text": ( + "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." + ) + }, + _run("release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff"), + ) + assert ok is True, note + + return asyncio.run(_go()) + + # --------------------------------------------------------------------------- # Prompt binding hard-fail + dry-run markers # --------------------------------------------------------------------------- diff --git a/tests/test_evals_output_contracts.py b/tests/test_evals_output_contracts.py new file mode 100644 index 00000000..3e36d596 --- /dev/null +++ b/tests/test_evals_output_contracts.py @@ -0,0 +1,119 @@ +"""Focused offline tests for the read-task line contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES +from evals.tasks import verify_c2, verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 + + +class _Page: + def __init__(self, results: list[Any]): + self.results = results + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str) -> dict[str, Any]: + return {"final_text": text, "calls": []} + + +def test_r2_written_number_prose_fails_and_count_contract_passes(): + async def _go(): + state = SimpleNamespace(id="started", name="Started", group="started") + items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] + plane = SimpleNamespace( + states=SimpleNamespace(list=lambda **kwargs: _Page([state])), + work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), + ) + ctx = {"workspace_slug": "ws", "project_id": "project"} + + prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) + contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) + + assert prose_ok is False + assert contract_ok is True, note + + return asyncio.run(_go()) + + +def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): + async def _go(): + overdue = "Session cookie not rotated after login" + ctx = { + "items": {R1_TITLE: "item-1", overdue: "item-2"}, + "r4_active_item_ids": ["item-1", "item-2"], + "r4_overdue_title": overdue, + } + text = f"cycle: {CYCLE_CURRENT}\nitem: {R1_TITLE}\nitem: {overdue}\noverdue: {overdue}" + + ok, note = await verify_r4(object(), ctx, _run(text)) + keyword_only_ok, _ = await verify_r4(object(), ctx, _run(f"cycle: {CYCLE_CURRENT}\noverdue")) + + assert ok is True, note + assert keyword_only_ok is False + + return asyncio.run(_go()) + + +def test_r5_exact_comment_lines_pass_but_free_prose_does_not(): + async def _go(): + ctx = {"r5_comment_phrases": list(R5_COMMENT_PHRASES)} + contract = "\n".join(f"comment: {phrase}" for phrase in reversed(R5_COMMENT_PHRASES)) + prose = f"The discussion covered {R5_COMMENT_PHRASES[0]} and {R5_COMMENT_PHRASES[1]}." + + contract_ok, note = await verify_r5(object(), ctx, _run(contract)) + prose_ok, _ = await verify_r5(object(), ctx, _run(prose)) + + assert contract_ok is True, note + assert prose_ok is False + + return asyncio.run(_go()) + + +def test_r6_exact_project_contract_passes_and_shorthand_fails(): + async def _go(): + expected = "EVAL deadbeef B" + ctx = {"r6_more_bugs_project": expected} + + exact_ok, note = await verify_r6(object(), ctx, _run(f"project: {expected}")) + shorthand_ok, _ = await verify_r6(object(), ctx, _run("The B project has more bugs.")) + + assert exact_ok is True, note + assert shorthand_ok is False + + return asyncio.run(_go()) + + +def test_r7_transition_contract_is_structural(): + async def _go(): + states = [ + SimpleNamespace(name="Backlog"), + SimpleNamespace(name="In Progress"), + SimpleNamespace(name="Done"), + ] + plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(states))) + ctx = {"workspace_slug": "ws", "project_id": "project"} + + exact_ok, note = await verify_r7(plane, ctx, _run("transition: Done")) + prose_ok, _ = await verify_r7(plane, ctx, _run("It can move to Done.")) + + assert exact_ok is True, note + assert prose_ok is False + + return asyncio.run(_go()) + + +def test_c2_correct_changelog_prose_without_contract_fails(): + async def _go(): + changelog = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." + prose = "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff." + + ok, _ = await verify_c2(object(), {"release_changelog_text": changelog}, _run(prose)) + + assert ok is False + + return asyncio.run(_go()) diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index 5cb68830..95838046 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -377,6 +377,26 @@ def test_report_main_table_cli(tmp_path: Path, capsys): assert "R1" in out +def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path, capsys): + f1 = tmp_path / "old.jsonl" + f2 = tmp_path / "new.jsonl" + f1.write_text( + json.dumps({**_synth_row("R1", surface="full"), "battery": "6425dcc64404"}) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps({**_synth_row("R1", surface="v2"), "battery": "newfinger001"}) + "\n", + encoding="utf-8", + ) + + rc = report_mod.main(["--table", str(f1), str(f2)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "spans battery fingerprints" in captured.err + assert "different task prompts/questions" in captured.err + + def test_report_main_markdown_flag(tmp_path: Path, capsys): f1 = tmp_path / "a.jsonl" f1.write_text(json.dumps(_synth_row("R1", surface="v2", num_calls=1)) + "\n", encoding="utf-8") diff --git a/tests/test_evals_verifiers.py b/tests/test_evals_verifiers.py index bc776d50..0e02fbb5 100644 --- a/tests/test_evals_verifiers.py +++ b/tests/test_evals_verifiers.py @@ -638,7 +638,24 @@ async def _go(): run, ) assert ok is False, note - assert "missing title" in note.lower() or "missing title(s)" in note + assert "item contract" in note.lower() + + return asyncio.run(_go()) + + +def test_minor_r3_exact_item_contract_passes_in_any_order(): + async def _go(): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + run = { + "final_text": ("item: Onboarding email template stale\nitem: Webhook secret rotation docs missing"), + "calls": [], + } + ok, note = await verify_r3( + object(), + {"r3_due_titles": titles, "r3_due_count": 2}, + run, + ) + assert ok is True, note return asyncio.run(_go()) From 9d7c759765dd3bf1b619a23c5f00cb424efcfa57 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 12:34:20 +0530 Subject: [PATCH 12/93] Aggregate repetitions instead of showing the last one Every battery so far has been n=1, and we have seen tasks flip between near-identical runs, so differences of one or two tasks are currently indistinguishable from variance. Measuring that noise floor needs the multi-rep path to be trustworthy, and an audit found it was not. Multi-file --table selected the last row per task and dropped earlier repetitions from cells and totals, so a five-rep battery would have rendered as one sample per task while looking like five. --reps 0 or negative ran nothing and exited successfully. A/B call-count comparison used the last successful repetition rather than a median, letting run order move the answer. Tables now aggregate every completed repetition, report per-task k/n with a Wilson interval, flag tasks that did not answer identically each time, and state the resulting minimum meaningful difference between surfaces. Single-rep files render exactly as before, which nearly all of our stored results are. Two caveats are documented rather than papered over: resume files do not record the intended rep count, so raising --reps works while lowering it leaves higher-numbered reps in place. Co-Authored-By: Claude Fable 5 --- evals/README.md | 8 ++ evals/cli.py | 4 + evals/report.py | 195 +++++++++++++++++++++++++-------- tests/test_evals_hardening.py | 63 ++++++++++- tests/test_evals_report_ops.py | 127 ++++++++++++++++++++- 5 files changed, 352 insertions(+), 45 deletions(-) diff --git a/evals/README.md b/evals/README.md index 6ffacb79..4f8ce1fd 100644 --- a/evals/README.md +++ b/evals/README.md @@ -144,6 +144,14 @@ denominators, as are rows with recorded errors. Result-token columns use `~` for `*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not recorded. +With `--reps N`, each `(task, rep)` is independently seeded, run, verified, and torn down. +Multi-rep reports show each task's pass count, Wilson interval, and whether its pass/fail +answer changed across completed repetitions. The measured noise-floor line converts those +flips into task-count units: if `U` tasks were unstable, surface differences of `U` tasks or +fewer should be treated as within observed run-to-run variance, making `U + 1` the minimum +meaningful difference from that sample. This is an empirical guardrail, not proof that larger +differences are statistically significant. + Every result row carries a `battery` fingerprint derived from the selected catalog's prompts and tool metadata. Compare rows only when their fingerprints match: a table that mixes fingerprints is comparing different questions, even when task IDs are the same. In particular, diff --git a/evals/cli.py b/evals/cli.py index 3b06c890..fc6c2c96 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -233,6 +233,10 @@ def main(argv: list[str] | None = None) -> int: if args.dry_run: return cmd_dry_run(tasks) + if args.reps < 1: + print("error: --reps must be at least 1", file=sys.stderr) + return 2 + surface = (args.surface or "full").strip().lower() server_cmd: list[str] | None = None if args.server_cmd: diff --git a/evals/report.py b/evals/report.py index 3784b128..e664f108 100644 --- a/evals/report.py +++ b/evals/report.py @@ -202,12 +202,14 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: by_task: dict[str, list[TaskResult]] = defaultdict(list) harness_err_by_task: dict[str, int] = defaultdict(int) infra_err_by_task: dict[str, int] = defaultdict(int) + reps_by_task: dict[str, set[int]] = defaultdict(set) infra_errors = 0 for raw_row in rows: r = _task_result(raw_row) if is_meta_row(r): continue tid = r.task_id + reps_by_task[tid].add(r.rep) if is_infra_error_row(r): infra_errors += 1 infra_err_by_task[tid] += 1 @@ -229,6 +231,7 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: trs = by_task.get(task_id, []) n = len(trs) k = sum(1 for r in trs if r.success) + unstable = n > 1 and 0 < k < n total_k += k total_n += n lo, hi = wilson_interval(k, n) if n else (0.0, 0.0) @@ -256,6 +259,7 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: "n": n, "k": k, "success": f"{k}/{n}" if n else "0/0", + "unstable": unstable, "wilson_lo": lo, "wilson_hi": hi, "med_calls": med_calls, @@ -275,12 +279,16 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: "med_cum_input": _median(cum_inputs), } agg_lo, agg_hi = wilson_interval(total_k, total_n) if total_n else (0.0, 0.0) + unstable_task_ids = sorted(task_id for task_id, values in out.items() if values.get("unstable")) out["_meta"] = { "infra_errors": infra_errors, "aggregate_k": total_k, "aggregate_n": total_n, "aggregate_wilson_lo": agg_lo, "aggregate_wilson_hi": agg_hi, + "multi_rep": any(len(reps) > 1 for reps in reps_by_task.values()), + "unstable_task_ids": unstable_task_ids, + "unstable_tasks": len(unstable_task_ids), "result_tokens_mode": result_tokens_mode([r for trs in by_task.values() for r in trs]), } return out @@ -303,6 +311,25 @@ def _fmt_result_tokens(x: float | None, mode: str) -> str: return f"{_result_tokens_marker(mode)}{value}" +def noise_floor_statement(unstable_tasks: int) -> str: + """Describe observed pass/fail variance in task-count comparison units.""" + count = max(0, int(unstable_tasks)) + if count == 0: + return ( + "measured noise floor: 0 tasks flipped at least once; no non-zero " + "run-to-run variance was observed (minimum meaningful difference " + "from observed flips: 1 task)" + ) + noun = "task" if count == 1 else "tasks" + threshold = count + 1 + threshold_noun = "task" if threshold == 1 else "tasks" + return ( + f"measured noise floor: {count} {noun} flipped at least once; surface " + f"differences of {count} {noun} or fewer are within observed run-to-run " + f"variance (minimum meaningful difference: {threshold} {threshold_noun})" + ) + + def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: meta = summary.get("_meta") or {} print(title) @@ -322,14 +349,20 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: ahi = float(meta.get("aggregate_wilson_hi") or 0.0) rate = agg_k / agg_n if agg_n else 0.0 print(f"aggregate success: {agg_k}/{agg_n} ({rate:.1%}) Wilson95 [{alo:.2f},{ahi:.2f}]") - # Show min/med/max call columns when any task has n>1. - show_var = any(s.get("n", 0) > 1 for tid, s in summary.items() if tid != "_meta") + multi_rep = bool(meta.get("multi_rep")) + if multi_rep: + print(noise_floor_statement(int(meta.get("unstable_tasks") or 0))) + # Multi-rep files keep the repetition-aware layout even when errors leave + # only one completed result in every task's success-rate denominator. + show_var = multi_rep or any(s.get("n", 0) > 1 for tid, s in summary.items() if tid != "_meta") token_marker = _result_tokens_marker(token_mode) med_rtok_header = f"med_rtok{token_marker}" p95_rtok_header = f"p95_rtok{token_marker}" if show_var: + unstable_header = f"{'unstable':>8} " if multi_rep else "" header = ( f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{unstable_header}" f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " f"{'IQR':>11} {'mispick':>8} {'err':>4} " f"{'capped':>6} {'h_err':>5} {'i_err':>5} {med_rtok_header:>9} {p95_rtok_header:>9} " @@ -352,8 +385,11 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: opt = s["optimal_calls"] if s["optimal_calls"] is not None else "-" task_token_mode = str(s.get("result_tokens_mode") or "unavailable") if show_var: + unstable = ("YES" if s.get("unstable") else "no") if multi_rep else "" + unstable_cell = f"{unstable:>8} " if multi_rep else "" print( f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " + f"{unstable_cell}" f"{_fmt(s.get('calls_min')):>9} {_fmt(s['med_calls']):>9} {_fmt(s.get('calls_max')):>9} " f"{opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " @@ -385,33 +421,32 @@ def ab_compare( ) -> dict[str, Any]: """Compare two result sets: paired call-count deltas + success rates. - Paired call deltas only include tasks that are present and successful in - both A and B. When multiple success rows exist for a task, the **last** one - wins (matches load-time ``dedupe="latest"`` semantics). + Paired call deltas only include tasks with at least one successful repetition + in both A and B. Calls are the median across successful repetitions; this is + identical to the historical behavior for single-rep files. """ sum_a = summarize(rows_a) sum_b = summarize(rows_b) - def _success_rows_by_task(rows: list[ResultRow]) -> dict[str, TaskResult]: - out: dict[str, TaskResult] = {} + def _success_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: + out: dict[str, list[float]] = defaultdict(list) for raw_row in rows: r = _task_result(raw_row) if is_meta_row(r) or is_infra_error_row(r) or r.error or r.skipped: continue if not r.success: continue - tid = r.task_id - out[tid] = r # last wins (dedupe already applied) - return out + out[r.task_id].append(float(r.num_calls)) + return dict(out) - sa = _success_rows_by_task(rows_a) - sb = _success_rows_by_task(rows_b) + sa = _success_calls_by_task(rows_a) + sb = _success_calls_by_task(rows_b) shared = sorted(set(sa) & set(sb)) deltas: list[float] = [] per_task: list[dict[str, Any]] = [] for tid in shared: - ca = float(sa[tid].num_calls) - cb = float(sb[tid].num_calls) + ca = float(_median(sa[tid]) or 0.0) + cb = float(_median(sb[tid]) or 0.0) d = cb - ca # B − A (negative = B fewer calls = better if lower is better) deltas.append(d) per_task.append({"task_id": tid, "calls_a": ca, "calls_b": cb, "delta": d}) @@ -425,6 +460,9 @@ def _success_rows_by_task(rows: list[ResultRow]) -> dict[str, TaskResult]: "median_delta": _median(deltas), "sign_test_p": sign_test_pvalue(deltas), "n_paired": len(deltas), + "multi_rep": bool(meta_a.get("multi_rep") or meta_b.get("multi_rep")), + "unstable_a": int(meta_a.get("unstable_tasks") or 0), + "unstable_b": int(meta_b.get("unstable_tasks") or 0), "success_a": { "k": int(meta_a.get("aggregate_k") or 0), "n": int(meta_a.get("aggregate_n") or 0), @@ -456,12 +494,19 @@ def print_ab_report(cmp: dict[str, Any], path_a: Path, path_b: Path) -> None: print(f" median call delta (B−A): {_fmt(cmp['median_delta'])}") p = cmp["sign_test_p"] print(f" sign-test p-value (two-sided): {p if p is not None else 'n/a'}") + multi_rep = bool(cmp.get("multi_rep")) + if multi_rep: + print(f" A {noise_floor_statement(int(cmp.get('unstable_a') or 0))}") + print(f" B {noise_floor_statement(int(cmp.get('unstable_b') or 0))}") if cmp["paired_tasks"]: print() print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") print("-" * 34) for row in cmp["paired_tasks"]: - print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") + if multi_rep: + print(f"{row['task_id']:<6} {_fmt(row['calls_a']):>8} {_fmt(row['calls_b']):>8} {row['delta']:>+8.1f}") + else: + print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") # --------------------------------------------------------------------------- @@ -498,6 +543,30 @@ def format_surface_cell(row: ResultRow | None) -> str: return f"{ok} {n_calls_s}c" +def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: + """Aggregate distinct repetitions into one task/surface cell.""" + results = [_task_result(row) for row in rows] + completed = [row for row in results if not is_infra_error_row(row) and not row.error and not row.skipped] + if completed: + n = len(completed) + k = sum(1 for row in completed if row.success) + lo, hi = wilson_interval(k, n) + if 0 < k < n: + marker = "⚠ UNSTABLE" + elif k == n: + marker = "✅" + else: + marker = "❌" + calls = [row.num_calls for row in completed] + call_span = f"{min(calls)}c" if min(calls) == max(calls) else f"{min(calls)}-{max(calls)}c" + return f"{marker} {k}/{n} [{lo:.2f},{hi:.2f}] {call_span}" + if any(row.error or is_infra_error_row(row) for row in results): + return "ERR" + if any(row.skipped for row in results): + return "skip" + return "—" + + def build_multi_surface_table( file_rows: list[tuple[str, list[ResultRow]]], ) -> dict[str, Any]: @@ -505,31 +574,39 @@ def build_multi_surface_table( ``file_rows`` is a list of ``(column_label, rows)``. Column labels default to each file's dominant ``surface`` field when the caller passes that label. - For each column, the latest row per task_id is used (rep-agnostic: last wins). + Rows are grouped by task and repetition. Single-rep columns retain the + historical one-cell rendering; multi-rep columns aggregate all repetitions. """ columns: list[str] = [] - by_col: dict[str, dict[str, TaskResult]] = {} + by_col: dict[str, dict[str, list[TaskResult]]] = {} + multi_rep_by_col: dict[str, bool] = {} for label, rows in file_rows: columns.append(label) - col_map: dict[str, TaskResult] = {} + col_map: dict[str, list[TaskResult]] = defaultdict(list) for raw_row in rows: if is_meta_row(raw_row): continue r = _task_result(raw_row) tid = r.task_id - col_map[tid] = r # last wins - by_col[label] = col_map + col_map[tid].append(r) + by_col[label] = dict(col_map) + multi_rep_by_col[label] = any(len({row.rep for row in task_rows}) > 1 for task_rows in col_map.values()) + + multi_rep = any(multi_rep_by_col.values()) all_tasks = sorted({t for m in by_col.values() for t in m}, key=_task_sort_key) cells: dict[str, dict[str, str]] = {} - raw: dict[str, dict[str, TaskResult | None]] = {} + raw: dict[str, dict[str, list[TaskResult]]] = {} for tid in all_tasks: cells[tid] = {} raw[tid] = {} for col in columns: - r = by_col[col].get(tid) - raw[tid][col] = r - cells[tid][col] = format_surface_cell(r) + task_rows = by_col[col].get(tid, []) + raw[tid][col] = task_rows + if multi_rep: + cells[tid][col] = format_multi_rep_surface_cell(task_rows) + else: + cells[tid][col] = format_surface_cell(task_rows[-1] if task_rows else None) # Aggregate footer per column. footer: dict[str, dict[str, Any]] = {} @@ -537,34 +614,51 @@ def build_multi_surface_table( succ = run = calls = mispicks = 0 mispick_comparable = True infra = 0 - for _tid, r in by_col[col].items(): - if is_infra_error_row(r): - infra += 1 - continue - if r.error: - continue - if r.skipped: - continue - run += 1 - if r.success: - succ += 1 - calls += r.num_calls - if r.classification == "external": - mispick_comparable = False - else: - alt, oos = r.alternate_calls, r.out_of_set_calls - if alt is None and oos is None: + unstable_tasks = 0 + for _tid, task_rows in by_col[col].items(): + completed: list[TaskResult] = [] + for r in task_rows: + if is_infra_error_row(r): + infra += 1 + continue + if r.error: + continue + if r.skipped: + continue + completed.append(r) + run += 1 + if r.success: + succ += 1 + calls += r.num_calls + if r.classification == "external": mispick_comparable = False else: - mispicks += int(alt or 0) + int(oos or 0) + alt, oos = r.alternate_calls, r.out_of_set_calls + if alt is None and oos is None: + mispick_comparable = False + else: + mispicks += int(alt or 0) + int(oos or 0) + task_k = sum(1 for r in completed if r.success) + if len(completed) > 1 and 0 < task_k < len(completed): + unstable_tasks += 1 footer[col] = { "success": succ, "n": run, "calls": calls, "mispicks": mispicks if mispick_comparable else None, "infra_errors": infra, + "multi_rep": multi_rep_by_col[col], + "unstable_tasks": unstable_tasks, } - return {"columns": columns, "task_ids": all_tasks, "cells": cells, "raw": raw, "footer": footer} + return { + "columns": columns, + "task_ids": all_tasks, + "cells": cells, + "raw": raw, + "footer": footer, + "multi_rep": multi_rep, + "multi_rep_by_col": multi_rep_by_col, + } def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) -> str: @@ -573,6 +667,7 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) task_ids: list[str] = table["task_ids"] cells: dict[str, dict[str, str]] = table["cells"] footer: dict[str, dict[str, Any]] = table["footer"] + multi_rep = bool(table.get("multi_rep")) lines: list[str] = [] def _prompt_snip(tid: str) -> str: @@ -595,9 +690,19 @@ def _prompt_snip(tid: str) -> str: mp = f", {f['mispicks']}mp" if f["mispicks"] is not None else "" foot_parts.append(f"{rate} ({f['calls']}c{mp}, i={f['infra_errors']})") lines.append("| **agg** | | " + " | ".join(foot_parts) + " |") + if multi_rep: + noise_parts = [ + noise_floor_statement(int(footer[c].get("unstable_tasks") or 0)) + if footer[c].get("multi_rep") + else "single repetition" + for c in cols + ] + lines.append("| **noise floor** | | " + " | ".join(noise_parts) + " |") return "\n".join(lines) + "\n" col_w = max(14, max((len(c) for c in cols), default=14)) + if multi_rep: + col_w = max(col_w, max((len(value) for task in cells.values() for value in task.values()), default=14)) head = f"{'task':5} {'what':34} " + " ".join(f"{c:{col_w}}" for c in cols) lines.append(head) lines.append("-" * len(head)) @@ -613,6 +718,10 @@ def _prompt_snip(tid: str) -> str: pct = f" ({100 * f['success'] / f['n']:.0f}%)" if f["n"] else "" mp = f" mispicks {f['mispicks']}" if f["mispicks"] is not None else " mispicks n/a" lines.append(f"{c:12} success {rate}{pct} total calls {f['calls']}{mp} infra {f['infra_errors']}") + if multi_rep: + for c in cols: + if footer[c].get("multi_rep"): + lines.append(f"{c:12} {noise_floor_statement(int(footer[c].get('unstable_tasks') or 0))}") return "\n".join(lines) + "\n" diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 55d6887d..6d9e9cfe 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -18,7 +18,7 @@ from evals import seed as seed_mod from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize -from evals.results import RESULT_SCHEMA_VERSION +from evals.results import RESULT_SCHEMA_VERSION, TaskResult from evals.run import ( is_infra_cli_stop_reason, load_resume_skip_keys, @@ -200,6 +200,11 @@ def test_parse_args_resume_and_canary(): assert b.canary is True +def test_live_run_rejects_non_positive_reps(capsys): + assert run_mod.main(["--tasks", "R1", "--reps", "0"]) == 2 + assert "--reps must be at least 1" in capsys.readouterr().err + + # --------------------------------------------------------------------------- # Error taxonomy (seed raise → infra_seed row) # --------------------------------------------------------------------------- @@ -907,6 +912,62 @@ def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): # --------------------------------------------------------------------------- +def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path: Path, monkeypatch): + out = tmp_path / "multi.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_ids: list[str] = [] + teardown_projects: list[str] = [] + + def fresh_seed(plane, run_id, needs, ctx): + seed_ids.append(run_id) + ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + + def record_teardown(plane, ctx): + teardown_projects.append(ctx["project_id"]) + + async def fake_agent(**kwargs): + return TaskResult(final_text="done", stop_reason="end_turn") + + monkeypatch.setattr(runner_mod, "seed", fresh_seed) + monkeypatch.setattr(runner_mod, "teardown", record_teardown) + monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kwargs: object()) + monkeypatch.setattr(runner_mod, "run_agent_task_via_driver", fake_agent) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + task = { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=3, + surface="full", + out_path=out, + driver_name="claude-cli", + ) + ) + + assert rc == 0 + assert len(seed_ids) == 3 + assert len(set(seed_ids)) == 3 + assert teardown_projects == seed_ids + rows = _data_rows(out) + assert [row["rep"] for row in rows] == [0, 1, 2] + assert all(row["success"] is True for row in rows) + + def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): out = tmp_path / "resume.jsonl" # Pre-write: completed R1/0 + infra R2/0 (same surface/battery/model/driver as this run). diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index 95838046..13b303a7 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -188,12 +188,79 @@ def test_summarize_aggregate_wilson_and_call_variance(): assert s["R1"]["calls_min"] == 2.0 assert s["R1"]["calls_max"] == 6.0 assert s["R1"]["med_calls"] == 4.0 + assert s["R1"]["unstable"] is True + assert s["R2"]["unstable"] is False meta = s["_meta"] assert meta["aggregate_k"] == 3 assert meta["aggregate_n"] == 4 + assert meta["multi_rep"] is True + assert meta["unstable_task_ids"] == ["R1"] + assert meta["unstable_tasks"] == 1 assert 0.0 <= meta["aggregate_wilson_lo"] <= meta["aggregate_wilson_hi"] <= 1.0 +def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): + path = tmp_path / "multi.jsonl" + outcomes = { + "R1": [True, True, True], + "R2": [True, False, True], + "R3": [False, False, False], + } + rows = [ + { + "task_id": task_id, + "rep": rep, + "surface": "full", + "success": success, + "num_calls": rep + 1, + "calls": [], + } + for task_id, task_outcomes in outcomes.items() + for rep, success in enumerate(task_outcomes) + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + loaded = load_rows(path) + summary = summarize(loaded) + + assert len(loaded) == 9 # distinct rep keys are not deduped away + assert summary["R1"]["success"] == "3/3" + assert summary["R1"]["unstable"] is False + assert summary["R2"]["success"] == "2/3" + assert summary["R2"]["wilson_lo"] == pytest.approx(0.2077, abs=1e-4) + assert summary["R2"]["wilson_hi"] == pytest.approx(0.9385, abs=1e-4) + assert summary["R2"]["unstable"] is True + assert summary["R3"]["success"] == "0/3" + assert summary["R3"]["unstable"] is False + assert summary["_meta"]["unstable_task_ids"] == ["R2"] + + report_mod.print_table(summary, "Summary: multi.jsonl") + output = capsys.readouterr().out + assert "unstable" in output + r2_line = next(line for line in output.splitlines() if line.startswith("R2")) + assert "2/3" in r2_line + assert "[0.21,0.94]" in r2_line + assert "YES" in r2_line + assert "measured noise floor: 1 task flipped at least once" in output + assert "minimum meaningful difference: 2 tasks" in output + + +def test_single_rep_summary_rendering_is_unchanged(capsys): + rows = [{"task_id": "R1", "rep": 0, "surface": "full", "success": True, "num_calls": 2, "calls": []}] + + report_mod.print_table(summarize(rows), "Summary: sample.jsonl") + + assert capsys.readouterr().out == ( + "Summary: sample.jsonl\n" + "aggregate success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" + "task n success wilson95 med_calls opt IQR mispick err capped h_err i_err " + "med_rtok p95_rtok med_cum_in\n" + "-------------------------------------------------------------------------------------------------------------------------------\n" + "R1 1 1/1 [0.21,1.00] 2.0 1 2.0-2.0 0.0% 0 0 0 0 " + "- - 0\n" + ) + + def test_report_marks_entirely_estimated_result_token_columns(capsys): rows = [ { @@ -272,6 +339,26 @@ def test_ab_compare_paired_deltas_and_sign_test(): assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 +def test_ab_compare_multi_rep_uses_median_successful_call_counts(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, + ] + + cmp = ab_compare(rows_a, rows_b) + + assert cmp["multi_rep"] is True + assert cmp["unstable_a"] == 1 + assert cmp["unstable_b"] == 0 + assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] + + # --------------------------------------------------------------------------- # Multi-surface table # --------------------------------------------------------------------------- @@ -280,6 +367,7 @@ def test_ab_compare_paired_deltas_and_sign_test(): def _synth_row( tid: str, *, + rep: int = 0, success: bool = True, num_calls: int = 2, alt: int | None = 0, @@ -292,7 +380,7 @@ def _synth_row( ) -> dict[str, Any]: return { "task_id": tid, - "rep": 0, + "rep": rep, "surface": surface, "success": success, "num_calls": num_calls, @@ -359,6 +447,43 @@ def test_multi_surface_table_snapshot_with_external(): assert table["footer"]["akhil"]["infra_errors"] == 1 +def test_multi_surface_table_aggregates_reps_and_flags_unstable(): + rows = [ + _synth_row("R1", rep=0, success=True, num_calls=2, surface="full"), + _synth_row("R1", rep=1, success=True, num_calls=3, surface="full"), + _synth_row("R1", rep=2, success=True, num_calls=2, surface="full"), + _synth_row("R2", rep=0, success=True, num_calls=1, surface="full"), + _synth_row("R2", rep=1, success=False, num_calls=4, surface="full"), + _synth_row("R2", rep=2, success=True, num_calls=2, surface="full"), + ] + + table = build_multi_surface_table([("full", rows)]) + + assert table["multi_rep"] is True + assert table["cells"]["R1"]["full"] == "✅ 3/3 [0.44,1.00] 2-3c" + assert table["cells"]["R2"]["full"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" + assert table["footer"]["full"]["success"] == 5 + assert table["footer"]["full"]["n"] == 6 + assert table["footer"]["full"]["unstable_tasks"] == 1 + rendered = render_multi_surface_table(table) + assert "measured noise floor: 1 task flipped at least once" in rendered + assert "minimum meaningful difference: 2 tasks" in rendered + + +def test_single_rep_multi_surface_rendering_is_unchanged(): + rows = [_synth_row("R1", surface="full", success=True, num_calls=2)] + + rendered = render_multi_surface_table(build_multi_surface_table([("full", rows)])) + + assert rendered == ( + "task what full \n" + "-------------------------------------------------------\n" + "R1 In project P, what is the curren… ✅ 2c\n" + "-------------------------------------------------------\n" + "full success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" + ) + + def test_report_main_table_cli(tmp_path: Path, capsys): f1 = tmp_path / "a.jsonl" f2 = tmp_path / "b.jsonl" From 87fb16e52aabcf6eb3ea377b96976a5f754a7018 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 13:07:54 +0530 Subject: [PATCH 13/93] Split the fixture builders by Plane object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seed.py was 1152 lines: client construction, the dry-run plan text, plan-gate detection, project creation with identifier retry, workspace and project feature toggles, the workspace-artifact preclean, the dispatcher, ten fixture builders and teardown. It was the last file in the harness where unrelated work shared a module. Each Plane object now owns its own module — work items, labels, item types, cycles, modules, intake, customers, releases — with the dispatcher in build.py, teardown in remove.py, and project setup in projects.py. The package takes over the evals.seed path, so no import anywhere needed editing, and __init__ carries re-exports and no logic. Names follow simple-technical-English rules: plain words with one meaning, no abbreviations, and nothing named utils, helpers, misc, common, core or manager, since a module whose name has no membership rule collects whatever has no home. modules.py is named for Plane's Module object and says so. Verified by the live canary rather than unit tests alone: it seeds and tears down every fixture path against a real API, still rejects a do-nothing agent on all 33 live tasks, and leaves no projects behind. Co-Authored-By: Claude Fable 5 --- evals/seed.py | 1152 -------------------------------------- evals/seed/__init__.py | 171 ++++++ evals/seed/build.py | 204 +++++++ evals/seed/client.py | 18 + evals/seed/customers.py | 31 + evals/seed/cycles.py | 137 +++++ evals/seed/intake.py | 42 ++ evals/seed/item_types.py | 79 +++ evals/seed/labels.py | 20 + evals/seed/modules.py | 61 ++ evals/seed/plan.py | 63 +++ evals/seed/projects.py | 245 ++++++++ evals/seed/releases.py | 34 ++ evals/seed/remove.py | 209 +++++++ evals/seed/work_items.py | 167 ++++++ 15 files changed, 1481 insertions(+), 1152 deletions(-) delete mode 100644 evals/seed.py create mode 100644 evals/seed/__init__.py create mode 100644 evals/seed/build.py create mode 100644 evals/seed/client.py create mode 100644 evals/seed/customers.py create mode 100644 evals/seed/cycles.py create mode 100644 evals/seed/intake.py create mode 100644 evals/seed/item_types.py create mode 100644 evals/seed/labels.py create mode 100644 evals/seed/modules.py create mode 100644 evals/seed/plan.py create mode 100644 evals/seed/projects.py create mode 100644 evals/seed/releases.py create mode 100644 evals/seed/remove.py create mode 100644 evals/seed/work_items.py diff --git a/evals/seed.py b/evals/seed.py deleted file mode 100644 index 891e6417..00000000 --- a/evals/seed.py +++ /dev/null @@ -1,1152 +0,0 @@ -"""Per-run fixture create/teardown via plane-sdk.""" - -from __future__ import annotations - -import os -import secrets -from datetime import date, timedelta -from typing import Any - -from plane import PlaneClient -from plane.errors.errors import HttpError -from plane.models.customers import CreateCustomer, CreateCustomerRequest -from plane.models.cycles import CreateCycle, UpdateCycle -from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest -from plane.models.labels import CreateLabel -from plane.models.modules import CreateModule -from plane.models.projects import CreateProject, ProjectFeature, UpdateProject -from plane.models.releases import CreateRelease, UpdateReleaseChangelog -from plane.models.work_item_types import CreateWorkItemType -from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem -from plane.models.workspaces import WorkspaceFeature - -# Soft-deleted projects reserve identifiers; create may 409 — retry with a new suffix. -_PROJECT_CREATE_MAX_ATTEMPTS = 3 - -# Fixed fixture titles for the `items` group. Exactly 4 urgent; the rest medium/high/low. -# "Payment webhook drops retries" is the R1 target (urgent, non-default state). -ITEM_FIXTURES: list[tuple[str, str]] = [ - ("Payment webhook drops retries", "urgent"), - ("Checkout times out on 3DS challenge", "urgent"), - ("Session cookie not rotated after login", "urgent"), - ("Inventory count goes negative under load", "urgent"), - ("Search results ignore archived projects", "high"), - ("CSV export truncates multi-byte chars", "high"), - ("Webhook secret rotation docs missing", "medium"), - ("Dark mode contrast fails WCAG AA", "medium"), - ("Onboarding email template stale", "medium"), - ("Sidebar collapse flickers on resize", "low"), - ("Tooltip clipped inside modal dialog", "low"), - ("Footer year still says 2024", "none"), -] - -R1_TITLE = ITEM_FIXTURES[0][0] -# R5 discussion target + distinctive comment phrases (word-boundary matched at verify). -R5_TITLE = "Checkout times out on 3DS challenge" -R5_COMMENT_PHRASES = ( - "stripe callback race", - "retry budget exhausted", -) -# W2 / W3 / W8 targets -W2_TITLE = "Sidebar collapse flickers on resize" -W3_TITLE = "Dark mode contrast fails WCAG AA" -W8_TITLE = R1_TITLE -# W7 relation pair + reference URL -W7_SOURCE_TITLE = "Search results ignore archived projects" -W7_TARGET_TITLE = "CSV export truncates multi-byte chars" -W7_URL = "https://example.com/eval/runbook-w7" -# R3: assignees + due this week (seeded count stored in ctx) -R3_DUE_TITLES = ( - "Webhook secret rotation docs missing", - "Onboarding email template stale", -) -# W6 unfinished items in Sprint 12 -W6_UNFINISHED_TITLES = ( - "Inventory count goes negative under load", - "Tooltip clipped inside modal dialog", -) -# Module completed-item titles (created extra when seeding module) -MODULE_NAME = "Checkout revamp" -MODULE_COMPLETED_TITLES = ( - "Module done: cart totals", - "Module done: tax lines", - "Module done: shipping quote", -) -# Intake fixtures -INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" -INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" -# Customer / release -CUSTOMER_NAME = "Acme Corp" -CUSTOMER_REQUEST_NAME = "SSO support" -RELEASE_NAME = "1.2.0" -RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." -# R6 second project bug titles -R6_MAIN_BUG_TITLES = ("Main bug alpha", "Main bug beta") -R6_SECOND_BUG_TITLES = ( - "Second bug one", - "Second bug two", - "Second bug three", - "Second bug four", -) - -LABEL_NAMES = ("auth", "triage", "perf") -CYCLE_PAST = "Sprint 12" -CYCLE_CURRENT = "Sprint 13" -# WS3 long-tail fixtures that live at workspace scope (must pre-clean + teardown). -DEBIAS_RELEASE_TAG_VERSION = "eval-rc1" -DEBIAS_CUSTOMER_PROP_DISPLAY = "Eval Industry" - - -def make_plane_client() -> tuple[PlaneClient, str]: - """Build a PlaneClient from EVAL_* env vars (mirrors stdio client construction).""" - api_key = os.environ.get("EVAL_PLANE_API_KEY", "") - workspace_slug = os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") - base_url = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") - if not api_key or not workspace_slug: - raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for live runs") - client = PlaneClient(base_url=base_url, api_key=api_key) - return client, workspace_slug - - -def seed_plan(needs: set[str]) -> list[str]: - """Human-readable seed plan for --dry-run (no network).""" - lines = [ - "project: EVAL {run8} (identifier EV{XXXX})", - ] - if "items" in needs: - lines.append(f"items: {len(ITEM_FIXTURES)} work items (exactly 4 urgent open)") - lines.append(f" - {R1_TITLE!r} (urgent, non-default started-group state) # R1 target") - lines.append(f" - {len(R3_DUE_TITLES)} assigned-to-me with due this week # R3") - lines.append(f" - comments on {R5_TITLE!r} # R5 discussion") - if "activity_feed" in needs: - lines.append( - f"activity_feed: gate that activities exist for {R5_TITLE!r} " - "(TaskSkipped env:no-activity-worker if empty) # L2" - ) - if "labels" in needs: - lines.append(f"labels: {', '.join(LABEL_NAMES)}") - if "bug_type" in needs: - lines.append( - "bug_type: work item type 'Bug' (genuine plan-gate only → skip dependents; other seed errors raise)" - ) - if "cycles" in needs: - past_state = "ends tomorrow, still OPEN so it can be closed" if "cycles_open_past" in needs else "past-dated" - lines.append(f"cycles: {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); unfinished on past") - if "module" in needs: - lines.append(f"module: {MODULE_NAME!r} with {len(MODULE_COMPLETED_TITLES)} completed items") - if "intake" in needs: - lines.append(f"intake: billing {INTAKE_BILLING_TITLE!r} + spam {INTAKE_SPAM_TITLE!r}") - if "customer" in needs: - lines.append(f"customer: {CUSTOMER_NAME!r} + request {CUSTOMER_REQUEST_NAME!r}") - if "release" in needs: - lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") - if "second_project" in needs: - lines.append("second_project: EVAL {run8} B with more open Bug-typed items than main (R6)") - if "leave_cycles_worklogs_off" in needs: - lines.append( - "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " - "(agent enables; teardown re-enables customers=True for later C1)" - ) - else: - lines.append( - "workspace_features: customers=True " - "(is_customer_enabled; NOT work_item_types — leaves S1/S3 type mode alone)" - ) - return lines - - -def _is_plan_gate(exc: BaseException) -> bool: - """True only for genuine plan/subscription feature gates — not generic API failures.""" - if not isinstance(exc, HttpError): - return False - if exc.status_code in (402, 403): - return True - blob = f"{exc} {exc.response!s}".lower() - keywords = ("plan", "subscription", "upgrade", "not available on your", "feature is not enabled") - return any(k in blob for k in keywords) - - -def is_identifier_collision(exc: BaseException) -> bool: - """True when project create failed because the identifier is already taken. - - Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare - ``identifier`` mention (validation shape errors) must not trigger retry. - """ - if not isinstance(exc, HttpError): - return False - if exc.status_code not in (400, 409): - return False - blob = f"{exc} {exc.response!s}".lower() - return any(k in blob for k in ("already", "exists", "taken")) - - -def create_project_with_identifier_retry( - plane: PlaneClient, - workspace_slug: str, - *, - name: str, - identifier_prefix: str, - initial_suffix: str, -) -> Any: - """Create a project, regenerating the identifier suffix on soft-delete collisions. - - Plane soft-deletes reserve identifiers; a 409 (or identifier-in-message error) - triggers a new random 4-char hex suffix. Max ``_PROJECT_CREATE_MAX_ATTEMPTS`` - attempts, then re-raises the last collision error. - """ - suffix = (initial_suffix or "")[:4].upper() - if len(suffix) < 4: - suffix = (suffix + secrets.token_hex(2).upper())[:4] - last_exc: BaseException | None = None - for attempt in range(_PROJECT_CREATE_MAX_ATTEMPTS): - if attempt > 0: - suffix = secrets.token_hex(2).upper() # 4 hex chars - identifier = f"{identifier_prefix}{suffix}" - try: - return plane.projects.create( - workspace_slug=workspace_slug, - data=CreateProject(name=name, identifier=identifier), - ) - except Exception as exc: - if is_identifier_collision(exc): - last_exc = exc - continue - raise - if last_exc is None: - raise RuntimeError( - f"project create failed after {_PROJECT_CREATE_MAX_ATTEMPTS} identifier retries " - f"(prefix={identifier_prefix!r}) with no captured exception" - ) - raise last_exc - - -def _enable_workspace_features( - plane: PlaneClient, - workspace_slug: str, - *, - exclude: set[str] | frozenset[str] | None = None, -) -> None: - """Enable workspace-level feature toggles that task preconditions need. - - Gate (plane-ee): create-customer 403 when - ``check_workspace_feature(slug, IS_CUSTOMER_ENABLED)`` is false — DB column - ``WorkspaceFeature.is_customer_enabled``. Legacy/SDK flips it via - ``workspaces.update_features`` / ``WorkspaceFeature(customers=True)`` - (API serializer maps ``customers`` → ``is_customer_enabled``). - - Deliberately does **not** set ``work_item_types``: that flips - workspace-vs-project type ownership and would change S1/S3 seed mode. - - ``exclude`` may contain ``customers`` (S5 leaves it off for the agent to enable). - """ - skip = set(exclude or ()) - data: dict[str, bool] = {} - if "customers" not in skip: - data["customers"] = True - if not data: - return - plane.workspaces.update_features( - workspace_slug=workspace_slug, - data=WorkspaceFeature(**data), - ) - - -def _enable_project_features( - plane: PlaneClient, - workspace_slug: str, - project_id: str, - *, - exclude: set[str] | frozenset[str] | None = None, -) -> None: - """Enable per-project feature gates that fresh projects ship with disabled. - - Two SDK calls (harmless if already on): - - 1. ``projects.update`` / ``UpdateProject`` — view columns API gates read: - ``cycle_view``, ``module_view``, ``intake_view``, ``page_view``, - ``is_time_tracking_enabled`` (worklog 404 when false). - 2. ``projects.update_features`` / ``ProjectFeature`` — capability flags - that the server maps onto the same view columns for cycles/modules/… . - - ``exclude`` is a set of feature keys to leave disabled (for S5): - ``cycles``, ``modules``, ``intakes``, ``pages``, ``worklogs``. - Default: enable all (other catalog tasks need them). - """ - skip = set(exclude or ()) - - upd_kwargs: dict[str, bool] = {} - if "cycles" not in skip: - upd_kwargs["cycle_view"] = True - if "modules" not in skip: - upd_kwargs["module_view"] = True - if "intakes" not in skip: - upd_kwargs["intake_view"] = True - if "pages" not in skip: - upd_kwargs["page_view"] = True - if "worklogs" not in skip: - upd_kwargs["is_time_tracking_enabled"] = True - if upd_kwargs: - plane.projects.update( - workspace_slug=workspace_slug, - project_id=project_id, - data=UpdateProject(**upd_kwargs), - ) - - feat_kwargs: dict[str, bool] = {} - if "cycles" not in skip: - feat_kwargs["cycles"] = True - if "modules" not in skip: - feat_kwargs["modules"] = True - if "intakes" not in skip: - feat_kwargs["intakes"] = True - if "pages" not in skip: - feat_kwargs["pages"] = True - if feat_kwargs: - plane.projects.update_features( - workspace_slug=workspace_slug, - project_id=project_id, - data=ProjectFeature(**feat_kwargs), - ) - - -def _preclean_ws3_workspace_artifacts(plane: PlaneClient, workspace_slug: str) -> None: - """Delete leftover WS3 long-tail artifacts so a dirty workspace cannot false-pass. - - Removes any existing release tag ``eval-rc1`` and customer property - ``Eval Industry`` before the rep seeds. - - Empty / not-found lists are silent. Clients without the API surface (offline - test stubs) are skipped silently. If a matching artifact is **found** and - cannot be deleted — or list fails on a present API — raises so the harness - records ``infra_seed`` rather than running against dirty state. - """ - releases = getattr(plane, "releases", None) - tags_api = getattr(releases, "tags", None) if releases is not None else None - if tags_api is not None: - try: - page = tags_api.list(workspace_slug=workspace_slug) - except Exception as exc: - raise RuntimeError(f"WS3 preclean: list release tags failed: {exc}") from exc - rows = page.results if hasattr(page, "results") else page - for tag in rows or []: - ver = (getattr(tag, "version", None) or "").strip() - if ver != DEBIAS_RELEASE_TAG_VERSION: - continue - tid = getattr(tag, "id", None) - if not tid: - continue - try: - tags_api.delete(workspace_slug=workspace_slug, tag_id=tid) - except Exception as exc: - raise RuntimeError( - f"WS3 preclean: failed to delete stale release tag {DEBIAS_RELEASE_TAG_VERSION!r} id={tid}: {exc}" - ) from exc - - customers = getattr(plane, "customers", None) - props_api = getattr(customers, "properties", None) if customers is not None else None - if props_api is not None: - try: - page = props_api.list(workspace_slug=workspace_slug) - except Exception as exc: - raise RuntimeError(f"WS3 preclean: list customer properties failed: {exc}") from exc - rows = page.results if hasattr(page, "results") else page - target = DEBIAS_CUSTOMER_PROP_DISPLAY.casefold() - for prop in rows or []: - display = (getattr(prop, "display_name", None) or getattr(prop, "name", None) or "").strip() - if display.casefold() != target: - continue - pid = getattr(prop, "id", None) - if not pid: - continue - try: - props_api.delete(workspace_slug=workspace_slug, property_id=pid) - except Exception as exc: - raise RuntimeError( - f"WS3 preclean: failed to delete stale customer property " - f"{DEBIAS_CUSTOMER_PROP_DISPLAY!r} id={pid}: {exc}" - ) from exc - - -def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) -> dict[str, Any]: - """Create the eval project and declared fixture groups. - - Mutates the caller-provided `ctx` in place so project_id is visible to teardown - even if a later fixture step raises (F5). - """ - run8 = run_id[:8] - project_name = f"EVAL {run8}" - workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] - - # Defensive: drop WS3 workspace artifacts that would make a no-op agent pass. - _preclean_ws3_workspace_artifacts(plane, workspace_slug) - - # Reset known keys while preserving object identity for the caller. - ctx.clear() - ctx.update( - { - "run_id": run_id, - "run8": run8, - "workspace_slug": workspace_slug, - "project_id": None, - "project_name": project_name, - "project_identifier": None, # filled after create (may retry suffix) - "labels": {}, - "items": {}, - "item_identifiers": {}, # title -> PROJ-N for ID-in-hand prompts - "item_ids": [], - "state_names": [], # all project state display names (for R1 negative check) - "r1_state_name": None, - "bug_type": None, - "bug_type_created": False, - "bug_type_workspace_level": False, - "bug_type_skip_reason": None, - "cycles": {}, - "module": None, - "module_completed_ids": [], - "intake": {}, - "customer": None, - "customer_request": None, - "release": None, - "second_project_id": None, - "second_project_name": None, - "r3_due_titles": list(R3_DUE_TITLES), - "r3_due_count": len(R3_DUE_TITLES), - "r5_title": R5_TITLE, - "r5_comment_phrases": list(R5_COMMENT_PHRASES), - "w6_unfinished_titles": list(W6_UNFINISHED_TITLES), - "workspace_objects": [], # [{kind, id}, ...] surviving project delete - } - ) - - # EV + 4 hex chars; retry with a new suffix on soft-delete identifier collisions. - project = create_project_with_identifier_retry( - plane, - workspace_slug, - name=project_name, - identifier_prefix="EV", - initial_suffix=run8[:4].upper(), - ) - ctx["project_id"] = project.id - ctx["project_identifier"] = getattr(project, "identifier", None) - - # Feature enablement (workspace first, then project). - # - # Ordering for S5 vs C1 on a shared eval workspace: - # - Each task-rep has its own seed/teardown; there is no multi-task seed batch. - # - Default tasks: enable workspace customers=True so C1 create_customer works. - # - S5 (needs leave_cycles_worklogs_off): leave project cycles+worklogs AND - # workspace customers OFF so the agent must flip all three; teardown then - # re-enables customers=True so a later C1 rep is not left 403ing. - # - We do not try to "run workspace enable after S5 check" — seed is per-task. - feature_exclude: set[str] = set() - ws_feature_exclude: set[str] = set() - if "leave_cycles_worklogs_off" in needs: - feature_exclude = {"cycles", "worklogs"} - ws_feature_exclude = {"customers"} - ctx["s5_left_customers_off"] = True - ctx["feature_exclude"] = sorted(feature_exclude) - ctx["ws_feature_exclude"] = sorted(ws_feature_exclude) - _enable_workspace_features(plane, workspace_slug, exclude=ws_feature_exclude) - _enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) - - # Labels before items so items can attach labels later if needed. - if "labels" in needs: - _seed_labels(plane, workspace_slug, ctx) - if "items" in needs: - _seed_items(plane, workspace_slug, ctx) - # L2: comments must materialize as activities (activity worker must be running). - if "activity_feed" in needs: - if "items" not in needs and not ctx.get("item_ids"): - _seed_items(plane, workspace_slug, ctx) - _gate_activity_worker(plane, workspace_slug, ctx) - if "bug_type" in needs: - _seed_bug_type(plane, workspace_slug, ctx) - if "cycles" in needs: - # Cycles need items to attach unfinished work; seed items if not already. - if "items" not in needs and not ctx["item_ids"]: - _seed_items(plane, workspace_slug, ctx) - _seed_cycles(plane, workspace_slug, ctx, leave_past_open="cycles_open_past" in needs) - if "module" in needs: - _seed_module(plane, workspace_slug, ctx) - if "intake" in needs: - _seed_intake(plane, workspace_slug, ctx) - if "customer" in needs: - _seed_customer(plane, workspace_slug, ctx) - if "release" in needs: - _seed_release(plane, workspace_slug, ctx) - if "second_project" in needs: - _seed_second_project(plane, workspace_slug, ctx) - - return ctx - - -def _seed_labels(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - for name in LABEL_NAMES: - label = plane.labels.create( - workspace_slug=workspace_slug, - project_id=ctx["project_id"], - data=CreateLabel(name=name), - ) - ctx["labels"][name] = label.id - - -def _list_states(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - return list(page.results or []) - - -def _completed_state(states: list[Any]) -> Any | None: - completed = [s for s in states if getattr(s, "group", None) == "completed"] - if not completed: - return None - # Prefer a non-default completed state named Done if present. - for s in completed: - if (s.name or "").strip().casefold() == "done": - return s - return completed[0] - - -def _seed_items(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - project_id = ctx["project_id"] - states = _list_states(plane, workspace_slug, project_id) - ctx["state_names"] = sorted({(s.name or "").strip() for s in states if (s.name or "").strip()}) - - # Prefer a non-default started-group state so R1 cannot be passed by guessing the default. - started = [s for s in states if getattr(s, "group", None) == "started" and not getattr(s, "default", False)] - if not started: - started = [s for s in states if getattr(s, "group", None) == "started"] - if not started: - raise RuntimeError( - "seed items: no started-group state available to place the R1 target; " - f"states={[(s.name, s.group, s.default) for s in states]}" - ) - r1_state = started[0] - ctx["r1_state_name"] = r1_state.name - ctx["r1_state_id"] = r1_state.id - - me = plane.users.get_me() - me_id = str(me.id) - ctx["me_id"] = me_id - # Due dates must stay inside the current ISO week (Mon–Sun). - # today+2d alone escapes the week on Sat/Sun — clamp to this week's Sunday. - today = date.today() - days_to_week_end = 6 - today.weekday() # Mon=0 … Sun=6 - due_this_week = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)).isoformat() - ctx["r3_due_date"] = due_this_week - - urgent_count = 0 - for title, priority in ITEM_FIXTURES: - data_kwargs: dict[str, Any] = {"name": title, "priority": priority} - if title == R1_TITLE: - data_kwargs["state"] = str(r1_state.id) - if title in R3_DUE_TITLES: - data_kwargs["assignees"] = [me_id] - data_kwargs["target_date"] = due_this_week - item = plane.work_items.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateWorkItem(**data_kwargs), # type: ignore[arg-type] - ) - # Some APIs ignore state on create; force via update if needed. - if title == R1_TITLE: - current = getattr(item, "state", None) - current_id = current if isinstance(current, str) else getattr(current, "id", None) - if str(current_id) != str(r1_state.id): - item = plane.work_items.update( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=item.id, - data=UpdateWorkItem(state=str(r1_state.id)), - ) - ctx["items"][title] = item.id - ctx["item_ids"].append(item.id) - seq = getattr(item, "sequence_id", None) - if seq is not None and ctx.get("project_identifier"): - ctx["item_identifiers"][title] = f"{ctx['project_identifier']}-{seq}" - if priority == "urgent": - urgent_count += 1 - assert urgent_count == 4, f"fixture invariant: expected 4 urgent items, got {urgent_count}" - - # R5: seed discussion comments on the known item. - r5_id = ctx["items"].get(R5_TITLE) - if r5_id: - for phrase in R5_COMMENT_PHRASES: - plane.work_items.comments.create( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=r5_id, - data=CreateWorkItemComment(comment_html=f"

{phrase}

"), - ) - - -def _gate_activity_worker(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - """Skip L2 when comments never materialize as activities (no activity worker). - - Raises :class:`evals.tasks.TaskSkipped` with reason ``env:no-activity-worker`` - so the harness records a skip, not a task failure. - """ - from evals.tasks import TaskSkipped - - project_id = ctx.get("project_id") - wid = (ctx.get("items") or {}).get(R5_TITLE) - if not project_id or not wid: - raise TaskSkipped("env:no-activity-worker") - try: - page = plane.work_items.activities.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=wid, - ) - except Exception as exc: - raise TaskSkipped(f"env:no-activity-worker ({type(exc).__name__}: {exc})") from exc - rows = page.results if hasattr(page, "results") else page - if len(list(rows or [])) < 1: - raise TaskSkipped("env:no-activity-worker") - - -def _seed_bug_type(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - """Create or resolve a 'Bug' work item type. - - Genuine plan-gate responses set bug_type=None + skip reason; all other failures raise. - Workspace feature probe uses the real key `is_work_item_types_enabled` (F10). - """ - project_id = ctx["project_id"] - target = "Bug" - try: - features = plane.workspaces.get_features(workspace_slug=workspace_slug) - dump = features.model_dump() if hasattr(features, "model_dump") else {} - # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional work_item_types key alone. - workspace_owns = bool(dump.get("is_work_item_types_enabled")) - - if workspace_owns: - existing = next( - ( - t - for t in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) - if (t.name or "").strip() == target - ), - None, - ) - created = False - if existing is None: - existing = plane.workspace_work_item_types.create( - workspace_slug=workspace_slug, data=CreateWorkItemType(name=target) - ) - created = True - plane.work_item_types.import_to_project( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_type_ids=[existing.id], - ) - ctx["bug_type"] = {"id": existing.id, "name": target} - ctx["bug_type_created"] = created - ctx["bug_type_workspace_level"] = True - if created: - ctx["workspace_objects"].append({"kind": "work_item_type", "id": existing.id}) - return - - # Per-project types. Project features expose no work-item-type toggle — do not PATCH. - existing = next( - ( - t - for t in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) - if (t.name or "").strip() == target - ), - None, - ) - created = False - if existing is None: - existing = plane.work_item_types.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateWorkItemType(name=target), - ) - created = True - ctx["bug_type"] = {"id": existing.id, "name": target} - ctx["bug_type_created"] = created - ctx["bug_type_workspace_level"] = False - except Exception as exc: - if _is_plan_gate(exc): - ctx["bug_type"] = None - ctx["bug_type_skip_reason"] = f"bug_type plan-gated: {exc}" - return - raise - - -def _seed_cycles(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any], leave_past_open: bool = False) -> None: - """Seed Sprint 12 (past) + Sprint 13 (active) with work items. - - Plane forbids adding issues to a cycle whose end_date is already past - (``The Cycle has already been completed so no new issues can be added`` — - plane-ee cycle/issue.py). Ordering for Sprint 12: - - 1. create with an *active* window (start past, end future) - 2. add_work_items while still active - 3. update end_date to the past (backdate) so the cycle is completed - - ``leave_past_open`` skips step 3, leaving Sprint 12 ending tomorrow. Closing a - cycle is only legal while it is still open — Plane rejects every edit to an - ended cycle (``The Cycle has already been completed so it cannot be edited``) - and rejects a transfer out of a still-running one (``The old cycle is not - completed yet``), so a fixture that pre-closes Sprint 12 makes "close it" - unachievable and leaves ``progress_snapshot`` (a transfer side effect) as the - only observable close signal. W6 asks the agent to close, so it seeds open. - - Sprint 13 is created and populated while genuinely active (start ≤ today ≤ end). - """ - project_id = ctx["project_id"] - me_id = ctx.get("me_id") or str(plane.users.get_me().id) - today = date.today() - # Final past window for Sprint 12 after backdate (completedCycles / W6 transfer source). - past_start = (today - timedelta(days=28)).isoformat() - past_end_final = (today - timedelta(days=14)).isoformat() - # Temporary active end so create + add succeed (end must be ≥ now). When the - # cycle stays open this is its final window, so keep it short — Sprint 12 ends - # tomorrow, which is what makes "close it and roll the rest over" natural. - past_end_active = (today + timedelta(days=1 if leave_past_open else 7)).isoformat() - # Sprint 13: genuinely active at seed time (start ≤ today ≤ end). - cur_start = (today - timedelta(days=3)).isoformat() - cur_end = (today + timedelta(days=10)).isoformat() - - # 1) Create Sprint 12 still active (items can be added). - past = plane.cycles.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateCycle( - name=CYCLE_PAST, - start_date=past_start, - end_date=past_end_active, - owned_by=me_id, - project_id=str(project_id), - ), - ) - # Sprint 13: active window for R4 / W6 transfer target. - current = plane.cycles.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateCycle( - name=CYCLE_CURRENT, - start_date=cur_start, - end_date=cur_end, - owned_by=me_id, - project_id=str(project_id), - ), - ) - ctx["cycles"] = { - CYCLE_PAST: past.id, - CYCLE_CURRENT: current.id, - } - ctx["cycle_past_id"] = past.id - ctx["cycle_current_id"] = current.id - - # 2) Add unfinished items to Sprint 12 *before* backdating. - unfinished_ids = [ctx["items"][t] for t in W6_UNFINISHED_TITLES if t in ctx["items"]] - if unfinished_ids: - plane.cycles.add_work_items( - workspace_slug=workspace_slug, - project_id=project_id, - cycle_id=past.id, - issue_ids=unfinished_ids, - ) - # R4: items on the active cycle (window still open). - active_ids: list[str] = [] - for title in (R1_TITLE, "Session cookie not rotated after login"): - iid = ctx["items"].get(title) - if iid: - active_ids.append(iid) - if active_ids: - plane.cycles.add_work_items( - workspace_slug=workspace_slug, - project_id=project_id, - cycle_id=current.id, - issue_ids=active_ids, - ) - overdue_id = ctx["items"].get("Session cookie not rotated after login") - if overdue_id: - plane.work_items.update( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=overdue_id, - data=UpdateWorkItem(target_date=(today - timedelta(days=3)).isoformat()), - ) - ctx["r4_overdue_title"] = "Session cookie not rotated after login" - ctx["r4_overdue_id"] = overdue_id - ctx["r4_active_item_ids"] = active_ids - - # 3) Backdate Sprint 12 so it is a completed cycle for R4 semantics — unless the - # task needs to close it itself, in which case it must still be open. - # UpdateCycle.end_date is writable; API allows past end_dates (no "can't backdate" gate - # on the update path — only add_work_items checks end_date < now). - if not leave_past_open: - plane.cycles.update( - workspace_slug=workspace_slug, - project_id=project_id, - cycle_id=past.id, - data=UpdateCycle(end_date=past_end_final), - ) - # Final seeded end_date for W6 close assertion (complete_cycle sets end_date=today). - ctx["cycle_past_seed_end_date"] = past_end_active if leave_past_open else past_end_final - ctx["cycle_past_open"] = leave_past_open - ctx["cycle_past_end_date_before_backdate"] = past_end_active - - -def _seed_module(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - project_id = ctx["project_id"] - states = _list_states(plane, workspace_slug, project_id) - done = _completed_state(states) - if done is None: - raise RuntimeError("seed module: no completed-group state to place module items") - - mod = plane.modules.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateModule(name=MODULE_NAME, status="in-progress"), - ) - ctx["module"] = {"id": mod.id, "name": MODULE_NAME} - completed_ids: list[str] = [] - for title in MODULE_COMPLETED_TITLES: - item = plane.work_items.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateWorkItem(name=title, priority="medium", state=str(done.id)), # type: ignore[arg-type] - ) - # Force completed state if create ignored it. - current = getattr(item, "state", None) - current_id = current if isinstance(current, str) else getattr(current, "id", None) - if str(current_id) != str(done.id): - item = plane.work_items.update( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=item.id, - data=UpdateWorkItem(state=str(done.id)), - ) - completed_ids.append(item.id) - ctx["items"][title] = item.id - ctx["item_ids"].append(item.id) - plane.modules.add_work_items( - workspace_slug=workspace_slug, - project_id=project_id, - module_id=mod.id, - issue_ids=completed_ids, - ) - ctx["module_completed_ids"] = completed_ids - ctx["module_completed_state_id"] = done.id - - -def _seed_intake(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - project_id = ctx["project_id"] - billing = plane.intake.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateIntakeWorkItem( - issue=WorkItemForIntakeRequest(name=INTAKE_BILLING_TITLE, priority="high"), - ), - ) - spam = plane.intake.create( - workspace_slug=workspace_slug, - project_id=project_id, - data=CreateIntakeWorkItem( - issue=WorkItemForIntakeRequest(name=INTAKE_SPAM_TITLE, priority="none"), - ), - ) - # IntakeWorkItem.issue is the work-item id used by triage tools. - ctx["intake"] = { - "billing": { - "intake_id": billing.id, - "issue_id": getattr(billing, "issue", None) or billing.id, - "title": INTAKE_BILLING_TITLE, - }, - "spam": { - "intake_id": spam.id, - "issue_id": getattr(spam, "issue", None) or spam.id, - "title": INTAKE_SPAM_TITLE, - }, - } - - -def _seed_customer(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - customer = plane.customers.create( - workspace_slug=workspace_slug, - data=CreateCustomer(name=CUSTOMER_NAME), - ) - ctx["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} - ctx["workspace_objects"].append({"kind": "customer", "id": customer.id}) - req = plane.customers.requests.create( - workspace_slug=workspace_slug, - customer_id=customer.id, - data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), - ) - ctx["customer_request"] = {"id": req.id, "name": CUSTOMER_REQUEST_NAME, "customer_id": customer.id} - - -def _seed_release(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - rel = plane.releases.create( - workspace_slug=workspace_slug, - data=CreateRelease(name=RELEASE_NAME), - ) - ctx["release"] = {"id": rel.id, "name": RELEASE_NAME} - ctx["workspace_objects"].append({"kind": "release", "id": rel.id}) - # Single changelog body; DESIGN's "2 entries" are encoded as plain-text bullets. - try: - plane.releases.changelog.update( - workspace_slug=workspace_slug, - release_id=rel.id, - data=UpdateReleaseChangelog( - description_html=f"

{RELEASE_CHANGELOG_TEXT}

", - ), - ) - except Exception as exc: - # Non-fatal for seed if changelog endpoint is flaky; C2 verifier still checks release name. - print(f"seed warning: release changelog update failed: {exc}") - ctx["release_changelog_text"] = RELEASE_CHANGELOG_TEXT - - -def _seed_second_project(plane: PlaneClient, workspace_slug: str, ctx: dict[str, Any]) -> None: - """R6: second project with more open Bug items than the main eval project.""" - run8 = ctx["run8"] - name = f"EVAL {run8} B" - project = create_project_with_identifier_retry( - plane, - workspace_slug, - name=name, - identifier_prefix="EB", - initial_suffix=run8[:4].upper(), - ) - ctx["second_project_id"] = project.id - ctx["second_project_name"] = name - ctx["second_project_identifier"] = getattr(project, "identifier", None) - # Track for teardown (project delete covers it; still record). - ctx["second_project_ids"] = [project.id] - _enable_project_features(plane, workspace_slug, project.id) - - # Ensure Bug type exists on both projects. - if not ctx.get("bug_type"): - _seed_bug_type(plane, workspace_slug, ctx) - bug = ctx.get("bug_type") or {} - bug_id = bug.get("id") if isinstance(bug, dict) else bug - if not bug_id: - raise RuntimeError("seed second_project: bug_type required for R6 bug counts") - - # Import workspace-level type into second project when needed. - if ctx.get("bug_type_workspace_level"): - try: - plane.work_item_types.import_to_project( - workspace_slug=workspace_slug, - project_id=project.id, - work_item_type_ids=[bug_id], - ) - except Exception as exc: - if not _is_plan_gate(exc): - # May already be imported. - if not (isinstance(exc, HttpError) and exc.status_code in (400, 409)): - raise - - main_id = ctx["project_id"] - # Main project: fewer bugs - main_bug_ids: list[str] = [] - for title in R6_MAIN_BUG_TITLES: - item = plane.work_items.create( - workspace_slug=workspace_slug, - project_id=main_id, - data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] - ) - main_bug_ids.append(item.id) - ctx["items"][title] = item.id - ctx["item_ids"].append(item.id) - # Second project: more bugs - second_bug_ids: list[str] = [] - for title in R6_SECOND_BUG_TITLES: - item = plane.work_items.create( - workspace_slug=workspace_slug, - project_id=project.id, - data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] - ) - second_bug_ids.append(item.id) - ctx["r6_main_bug_count"] = len(main_bug_ids) - ctx["r6_second_bug_count"] = len(second_bug_ids) - ctx["r6_more_bugs_project"] = name # second project has more - - -def _cleanup_severity_on_bug_type(plane: PlaneClient, ctx: dict[str, Any]) -> None: - """Delete Severity properties attached to the seeded Bug type (avoids multi-rep pollution).""" - bug = ctx.get("bug_type") - if not bug: - return - bug_type_id = bug.get("id") if isinstance(bug, dict) else bug - if not bug_type_id: - return - workspace_slug = ctx.get("workspace_slug") or "" - project_id = ctx.get("project_id") - - props: list[Any] = [] - try: - if project_id: - props = list( - plane.work_item_properties.list( - workspace_slug=workspace_slug, - project_id=project_id, - type_id=str(bug_type_id), - ) - or [] - ) - except HttpError as exc: - if exc.status_code not in (404, 405): - print(f"teardown warning: list Severity props failed: {exc}") - return - except Exception as exc: - print(f"teardown warning: list Severity props failed: {exc}") - return - - for p in props: - display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() - if display.lower() != "severity": - continue - try: - if project_id: - plane.work_item_properties.delete( - workspace_slug=workspace_slug, - project_id=project_id, - type_id=str(bug_type_id), - work_item_property_id=p.id, - ) - ctx.setdefault("workspace_objects", []) # no-op anchor - except Exception as exc: - print(f"teardown warning: failed to delete Severity property {p.id}: {exc}") - - -def _cleanup_agent_incident_type(plane: PlaneClient, ctx: dict[str, Any]) -> None: - """Best-effort cleanup of agent-created Incident type (S3 multi-rep pollution).""" - workspace_slug = ctx.get("workspace_slug") or "" - project_id = ctx.get("project_id") - try: - if ctx.get("bug_type_workspace_level"): - for t in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []: - if (t.name or "").strip().casefold() == "incident": - plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=t.id) - elif project_id: - for t in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []: - if (t.name or "").strip().casefold() == "incident": - plane.work_item_types.delete( - workspace_slug=workspace_slug, project_id=project_id, work_item_type_id=t.id - ) - except Exception as exc: - print(f"teardown warning: Incident type cleanup failed: {exc}") - - -def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: - """Delete the project and any workspace-scoped objects we created.""" - if not ctx: - return - workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") - project_id = ctx.get("project_id") - - # S5 left customers off (or agent enabled them): re-enable for subsequent task-reps - # on the shared eval workspace. We always set customers=True — we do not restore a - # prior false (S5's job is to leave the workspace usable for C1). - if ctx.get("s5_left_customers_off"): - try: - plane.workspaces.update_features( - workspace_slug=workspace_slug, - data=WorkspaceFeature(customers=True), - ) - except Exception as exc: - print(f"teardown warning: re-enable workspace customers failed: {exc}") - - # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). - try: - _cleanup_severity_on_bug_type(plane, ctx) - except Exception as exc: - print(f"teardown warning: Severity cleanup failed: {exc}") - try: - _cleanup_agent_incident_type(plane, ctx) - except Exception as exc: - print(f"teardown warning: Incident cleanup failed: {exc}") - - # Best-effort: agent-created Acme Corp customers (C1) that never hit workspace_objects. - try: - page = plane.customers.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page - for c in rows or []: - if (c.name or "").strip().casefold() in (CUSTOMER_NAME.casefold(), "acme"): - # Only delete if we seeded or created during this run (tracked or name match + run). - tracked = {o.get("id") for o in (ctx.get("workspace_objects") or []) if o.get("kind") == "customer"} - if str(c.id) in tracked or ctx.get("customer") is None: - # Avoid deleting long-lived Acme if we pre-seeded and tracked it — still delete tracked. - if str(c.id) in tracked or not ctx.get("customer"): - ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": c.id}) - except Exception as exc: - print(f"teardown warning: customer scan failed: {exc}") - - # Workspace-scoped cleanup first (survive project deletion). - seen_ws: set[str] = set() - for obj in ctx.get("workspace_objects") or []: - kind = obj.get("kind") - oid = obj.get("id") - if not oid: - continue - key = f"{kind}:{oid}" - if key in seen_ws: - continue - seen_ws.add(key) - try: - if kind == "work_item_type": - plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=oid) - elif kind == "work_item_property": - plane.workspace_work_item_properties.delete(workspace_slug=workspace_slug, property_id=oid) - elif kind == "customer": - plane.customers.delete(workspace_slug=workspace_slug, customer_id=oid) - elif kind == "release": - plane.releases.delete(workspace_slug=workspace_slug, release_id=oid) - elif kind == "release_tag": - plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=oid) - elif kind == "customer_property": - plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=oid) - except Exception as exc: - print(f"teardown warning: failed to delete workspace {kind} {oid}: {exc}") - - # Sweep by well-known WS3 names in case tracking missed an agent-created row. - try: - page = plane.releases.tags.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page - for tag in rows or []: - if (getattr(tag, "version", None) or "").strip() == DEBIAS_RELEASE_TAG_VERSION: - tid = getattr(tag, "id", None) - if tid and f"release_tag:{tid}" not in seen_ws: - try: - plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tid) - except Exception as exc: - print(f"teardown warning: sweep release tag {tid}: {exc}") - except Exception as exc: - print(f"teardown warning: sweep release tags failed: {exc}") - try: - page = plane.customers.properties.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page - target = DEBIAS_CUSTOMER_PROP_DISPLAY.casefold() - for prop in rows or []: - display = (getattr(prop, "display_name", None) or getattr(prop, "name", None) or "").strip() - if display.casefold() == target: - pid = getattr(prop, "id", None) - if pid and f"customer_property:{pid}" not in seen_ws: - try: - plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=pid) - except Exception as exc: - print(f"teardown warning: sweep customer property {pid}: {exc}") - except Exception as exc: - print(f"teardown warning: sweep customer properties failed: {exc}") - - # Second project before main (no dependency either way, but be thorough). - for pid in ctx.get("second_project_ids") or []: - if not pid or pid == project_id: - continue - try: - plane.projects.delete(workspace_slug=workspace_slug, project_id=pid) - except Exception as exc: - print(f"teardown warning: failed to delete second project {pid}: {exc}") - - if project_id: - try: - plane.projects.delete(workspace_slug=workspace_slug, project_id=project_id) - except Exception as exc: - name = ctx.get("project_name", project_id) - print(f"teardown warning: failed to delete project {name!r}: {exc}") - print(f"orphaned project: {name}") diff --git a/evals/seed/__init__.py b/evals/seed/__init__.py new file mode 100644 index 00000000..93904abf --- /dev/null +++ b/evals/seed/__init__.py @@ -0,0 +1,171 @@ +"""Evaluation fixture creation and removal.""" + +from .build import remove_stale_workspace_artifacts, seed +from .build import remove_stale_workspace_artifacts as _preclean_ws3_workspace_artifacts +from .client import make_plane_client +from .customers import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + seed_customer, +) +from .customers import ( + EVALUATION_CUSTOMER_PROPERTY_NAME as DEBIAS_CUSTOMER_PROP_DISPLAY, +) +from .cycles import CYCLE_CURRENT, CYCLE_PAST, seed_cycles +from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, seed_intake +from .item_types import seed_item_type +from .labels import LABEL_NAMES, seed_labels +from .modules import MODULE_COMPLETED_TITLES, MODULE_NAME, seed_module +from .plan import seed_plan +from .projects import ( + MAIN_PROJECT_BUG_TITLES, + SECOND_PROJECT_BUG_TITLES, + create_project_with_identifier_retry, + enable_project_features, + enable_workspace_features, + is_identifier_collision, + is_plan_gate, + secrets, + seed_second_project, +) +from .projects import ( + MAIN_PROJECT_BUG_TITLES as R6_MAIN_BUG_TITLES, +) +from .projects import ( + SECOND_PROJECT_BUG_TITLES as R6_SECOND_BUG_TITLES, +) +from .releases import ( + EVALUATION_RELEASE_TAG_VERSION, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, + seed_release, +) +from .releases import ( + EVALUATION_RELEASE_TAG_VERSION as DEBIAS_RELEASE_TAG_VERSION, +) +from .remove import teardown +from .work_items import ( + BLOCKING_REFERENCE_ADDRESS, + BLOCKING_SOURCE_TITLE, + BLOCKING_TARGET_TITLE, + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DARK_MODE_TITLE, + DUE_THIS_WEEK_TITLES, + PAYMENT_WEBHOOK_TITLE, + SIDEBAR_TITLE, + UNFINISHED_CYCLE_TITLES, + WORK_ITEM_FIXTURES, + find_completed_state, + list_states, + require_activities, + seed_work_items, +) +from .work_items import ( + BLOCKING_REFERENCE_ADDRESS as W7_URL, +) +from .work_items import ( + BLOCKING_SOURCE_TITLE as W7_SOURCE_TITLE, +) +from .work_items import ( + BLOCKING_TARGET_TITLE as W7_TARGET_TITLE, +) +from .work_items import ( + CHECKOUT_COMMENT_PHRASES as R5_COMMENT_PHRASES, +) +from .work_items import ( + CHECKOUT_TIMEOUT_TITLE as R5_TITLE, +) +from .work_items import ( + DARK_MODE_TITLE as W3_TITLE, +) +from .work_items import ( + DUE_THIS_WEEK_TITLES as R3_DUE_TITLES, +) +from .work_items import ( + PAYMENT_WEBHOOK_TITLE as R1_TITLE, +) +from .work_items import ( + PAYMENT_WEBHOOK_TITLE as W8_TITLE, +) +from .work_items import ( + SIDEBAR_TITLE as W2_TITLE, +) +from .work_items import ( + UNFINISHED_CYCLE_TITLES as W6_UNFINISHED_TITLES, +) +from .work_items import ( + WORK_ITEM_FIXTURES as ITEM_FIXTURES, +) +from .work_items import require_activities as _gate_activity_worker + +__all__ = [ + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "CYCLE_CURRENT", + "CYCLE_PAST", + "DEBIAS_CUSTOMER_PROP_DISPLAY", + "DEBIAS_RELEASE_TAG_VERSION", + "DARK_MODE_TITLE", + "DUE_THIS_WEEK_TITLES", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "EVALUATION_RELEASE_TAG_VERSION", + "INTAKE_BILLING_TITLE", + "INTAKE_SPAM_TITLE", + "ITEM_FIXTURES", + "LABEL_NAMES", + "MAIN_PROJECT_BUG_TITLES", + "MODULE_COMPLETED_TITLES", + "MODULE_NAME", + "PAYMENT_WEBHOOK_TITLE", + "R1_TITLE", + "R3_DUE_TITLES", + "R5_COMMENT_PHRASES", + "R5_TITLE", + "R6_MAIN_BUG_TITLES", + "R6_SECOND_BUG_TITLES", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "SECOND_PROJECT_BUG_TITLES", + "SIDEBAR_TITLE", + "UNFINISHED_CYCLE_TITLES", + "W2_TITLE", + "W3_TITLE", + "W6_UNFINISHED_TITLES", + "W7_SOURCE_TITLE", + "W7_TARGET_TITLE", + "W7_URL", + "W8_TITLE", + "WORK_ITEM_FIXTURES", + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "_gate_activity_worker", + "_preclean_ws3_workspace_artifacts", + "create_project_with_identifier_retry", + "enable_project_features", + "enable_workspace_features", + "find_completed_state", + "is_identifier_collision", + "is_plan_gate", + "list_states", + "make_plane_client", + "remove_stale_workspace_artifacts", + "require_activities", + "secrets", + "seed", + "seed_customer", + "seed_cycles", + "seed_intake", + "seed_item_type", + "seed_labels", + "seed_module", + "seed_plan", + "seed_release", + "seed_second_project", + "seed_work_items", + "teardown", +] diff --git a/evals/seed/build.py b/evals/seed/build.py new file mode 100644 index 00000000..512de3bf --- /dev/null +++ b/evals/seed/build.py @@ -0,0 +1,204 @@ +"""Fixture dispatch and workspace preclean for evaluation runs.""" + +from __future__ import annotations + +import os +from typing import Any + +from plane import PlaneClient + +from .customers import EVALUATION_CUSTOMER_PROPERTY_NAME, seed_customer +from .cycles import seed_cycles +from .intake import seed_intake +from .item_types import seed_item_type +from .labels import seed_labels +from .modules import seed_module +from .projects import ( + create_project_with_identifier_retry, + enable_project_features, + enable_workspace_features, + seed_second_project, +) +from .releases import EVALUATION_RELEASE_TAG_VERSION, seed_release +from .work_items import ( + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DUE_THIS_WEEK_TITLES, + UNFINISHED_CYCLE_TITLES, + require_activities, + seed_work_items, +) + + +def remove_stale_workspace_artifacts(plane: PlaneClient, workspace_slug: str) -> None: + """Delete leftover WS3 long-tail artifacts so a dirty workspace cannot false-pass. + + Removes any existing release tag ``eval-rc1`` and customer property + ``Eval Industry`` before the rep seeds. + + Empty / not-found lists are silent. Clients without the API surface (offline + test stubs) are skipped silently. If a matching artifact is **found** and + cannot be deleted — or list fails on a present API — raises so the harness + records ``infra_seed`` rather than running against dirty state. + """ + releases = getattr(plane, "releases", None) + tags_api = getattr(releases, "tags", None) if releases is not None else None + if tags_api is not None: + try: + page = tags_api.list(workspace_slug=workspace_slug) + except Exception as exc: + raise RuntimeError(f"WS3 preclean: list release tags failed: {exc}") from exc + rows = page.results if hasattr(page, "results") else page + for tag in rows or []: + version = (getattr(tag, "version", None) or "").strip() + if version != EVALUATION_RELEASE_TAG_VERSION: + continue + tag_id = getattr(tag, "id", None) + if not tag_id: + continue + try: + tags_api.delete(workspace_slug=workspace_slug, tag_id=tag_id) + except Exception as exc: + raise RuntimeError( + f"WS3 preclean: failed to delete stale release tag " + f"{EVALUATION_RELEASE_TAG_VERSION!r} id={tag_id}: {exc}" + ) from exc + + customers = getattr(plane, "customers", None) + properties_api = getattr(customers, "properties", None) if customers is not None else None + if properties_api is not None: + try: + page = properties_api.list(workspace_slug=workspace_slug) + except Exception as exc: + raise RuntimeError(f"WS3 preclean: list customer properties failed: {exc}") from exc + rows = page.results if hasattr(page, "results") else page + target = EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + for customer_property in rows or []: + display = ( + getattr(customer_property, "display_name", None) or getattr(customer_property, "name", None) or "" + ).strip() + if display.casefold() != target: + continue + property_id = getattr(customer_property, "id", None) + if not property_id: + continue + try: + properties_api.delete(workspace_slug=workspace_slug, property_id=property_id) + except Exception as exc: + raise RuntimeError( + f"WS3 preclean: failed to delete stale customer property " + f"{EVALUATION_CUSTOMER_PROPERTY_NAME!r} id={property_id}: {exc}" + ) from exc + + +def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) -> dict[str, Any]: + """Create the eval project and declared fixture groups. + + Mutates the caller-provided `ctx` in place so project_id is visible to teardown + even if a later fixture step raises (F5). + """ + run_prefix = run_id[:8] + project_name = f"EVAL {run_prefix}" + workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + + # Defensive: drop WS3 workspace artifacts that would make a no-op agent pass. + remove_stale_workspace_artifacts(plane, workspace_slug) + + # Reset known keys while preserving object identity for the caller. + ctx.clear() + ctx.update( + { + "run_id": run_id, + "run8": run_prefix, + "workspace_slug": workspace_slug, + "project_id": None, + "project_name": project_name, + "project_identifier": None, # filled after create (may retry suffix) + "labels": {}, + "items": {}, + "item_identifiers": {}, # title -> PROJ-N for ID-in-hand prompts + "item_ids": [], + "state_names": [], # all project state display names (for R1 negative check) + "r1_state_name": None, + "bug_type": None, + "bug_type_created": False, + "bug_type_workspace_level": False, + "bug_type_skip_reason": None, + "cycles": {}, + "module": None, + "module_completed_ids": [], + "intake": {}, + "customer": None, + "customer_request": None, + "release": None, + "second_project_id": None, + "second_project_name": None, + "r3_due_titles": list(DUE_THIS_WEEK_TITLES), + "r3_due_count": len(DUE_THIS_WEEK_TITLES), + "r5_title": CHECKOUT_TIMEOUT_TITLE, + "r5_comment_phrases": list(CHECKOUT_COMMENT_PHRASES), + "w6_unfinished_titles": list(UNFINISHED_CYCLE_TITLES), + "workspace_objects": [], # [{kind, id}, ...] surviving project delete + } + ) + + # EV + 4 hex chars; retry with a new suffix on soft-delete identifier collisions. + project = create_project_with_identifier_retry( + plane, + workspace_slug, + name=project_name, + identifier_prefix="EV", + initial_suffix=run_prefix[:4].upper(), + ) + ctx["project_id"] = project.id + ctx["project_identifier"] = getattr(project, "identifier", None) + + # Feature enablement (workspace first, then project). + # + # Ordering for S5 vs C1 on a shared eval workspace: + # - Each task-rep has its own seed/teardown; there is no multi-task seed batch. + # - Default tasks: enable workspace customers=True so C1 create_customer works. + # - S5 (needs leave_cycles_worklogs_off): leave project cycles+worklogs AND + # workspace customers OFF so the agent must flip all three; teardown then + # re-enables customers=True so a later C1 rep is not left 403ing. + # - We do not try to "run workspace enable after S5 check" — seed is per-task. + feature_exclude: set[str] = set() + workspace_feature_exclude: set[str] = set() + if "leave_cycles_worklogs_off" in needs: + feature_exclude = {"cycles", "worklogs"} + workspace_feature_exclude = {"customers"} + ctx["s5_left_customers_off"] = True + ctx["feature_exclude"] = sorted(feature_exclude) + ctx["ws_feature_exclude"] = sorted(workspace_feature_exclude) + enable_workspace_features(plane, workspace_slug, exclude=workspace_feature_exclude) + enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) + + # Labels before items so items can attach labels later if needed. + if "labels" in needs: + seed_labels(plane, workspace_slug, ctx) + if "items" in needs: + seed_work_items(plane, workspace_slug, ctx) + # L2: comments must materialize as activities (activity worker must be running). + if "activity_feed" in needs: + if "items" not in needs and not ctx.get("item_ids"): + seed_work_items(plane, workspace_slug, ctx) + require_activities(plane, workspace_slug, ctx) + if "bug_type" in needs: + seed_item_type(plane, workspace_slug, ctx) + if "cycles" in needs: + # Cycles need items to attach unfinished work; seed items if not already. + if "items" not in needs and not ctx["item_ids"]: + seed_work_items(plane, workspace_slug, ctx) + seed_cycles(plane, workspace_slug, ctx, leave_past_open="cycles_open_past" in needs) + if "module" in needs: + seed_module(plane, workspace_slug, ctx) + if "intake" in needs: + seed_intake(plane, workspace_slug, ctx) + if "customer" in needs: + seed_customer(plane, workspace_slug, ctx) + if "release" in needs: + seed_release(plane, workspace_slug, ctx) + if "second_project" in needs: + seed_second_project(plane, workspace_slug, ctx) + + return ctx diff --git a/evals/seed/client.py b/evals/seed/client.py new file mode 100644 index 00000000..9d14b3ed --- /dev/null +++ b/evals/seed/client.py @@ -0,0 +1,18 @@ +"""Plane client construction for evaluation fixture runs.""" + +from __future__ import annotations + +import os + +from plane import PlaneClient + + +def make_plane_client() -> tuple[PlaneClient, str]: + """Build a PlaneClient from EVAL_* env vars (mirrors stdio client construction).""" + api_key = os.environ.get("EVAL_PLANE_API_KEY", "") + workspace_slug = os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + base_url = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if not api_key or not workspace_slug: + raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for live runs") + client = PlaneClient(base_url=base_url, api_key=api_key) + return client, workspace_slug diff --git a/evals/seed/customers.py b/evals/seed/customers.py new file mode 100644 index 00000000..6ae73edc --- /dev/null +++ b/evals/seed/customers.py @@ -0,0 +1,31 @@ +"""Customer fixtures for evaluation workspaces.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.customers import CreateCustomer, CreateCustomerRequest + +CUSTOMER_NAME = "Acme Corp" +CUSTOMER_REQUEST_NAME = "SSO support" +EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" + + +def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + customer = plane.customers.create( + workspace_slug=workspace_slug, + data=CreateCustomer(name=CUSTOMER_NAME), + ) + context["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} + context["workspace_objects"].append({"kind": "customer", "id": customer.id}) + request = plane.customers.requests.create( + workspace_slug=workspace_slug, + customer_id=customer.id, + data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), + ) + context["customer_request"] = { + "id": request.id, + "name": CUSTOMER_REQUEST_NAME, + "customer_id": customer.id, + } diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py new file mode 100644 index 00000000..65841279 --- /dev/null +++ b/evals/seed/cycles.py @@ -0,0 +1,137 @@ +"""Cycle fixtures for evaluation projects.""" + +from __future__ import annotations + +from datetime import date, timedelta +from typing import Any + +from plane import PlaneClient +from plane.models.cycles import CreateCycle, UpdateCycle +from plane.models.work_items import UpdateWorkItem + +from .work_items import PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES + +CYCLE_PAST = "Sprint 12" +CYCLE_CURRENT = "Sprint 13" + + +def seed_cycles( + plane: PlaneClient, + workspace_slug: str, + context: dict[str, Any], + leave_past_open: bool = False, +) -> None: + """Seed Sprint 12 (past) + Sprint 13 (active) with work items. + + Plane forbids adding issues to a cycle whose end_date is already past + (``The Cycle has already been completed so no new issues can be added`` — + plane-ee cycle/issue.py). Ordering for Sprint 12: + + 1. create with an *active* window (start past, end future) + 2. add_work_items while still active + 3. update end_date to the past (backdate) so the cycle is completed + + ``leave_past_open`` skips step 3, leaving Sprint 12 ending tomorrow. Closing a + cycle is only legal while it is still open — Plane rejects every edit to an + ended cycle (``The Cycle has already been completed so it cannot be edited``) + and rejects a transfer out of a still-running one (``The old cycle is not + completed yet``), so a fixture that pre-closes Sprint 12 makes "close it" + unachievable and leaves ``progress_snapshot`` (a transfer side effect) as the + only observable close signal. W6 asks the agent to close, so it seeds open. + + Sprint 13 is created and populated while genuinely active (start ≤ today ≤ end). + """ + project_id = context["project_id"] + me_id = context.get("me_id") or str(plane.users.get_me().id) + today = date.today() + # Final past window for Sprint 12 after backdate (completedCycles / W6 transfer source). + past_start = (today - timedelta(days=28)).isoformat() + past_end_final = (today - timedelta(days=14)).isoformat() + # Temporary active end so create + add succeed (end must be ≥ now). When the + # cycle stays open this is its final window, so keep it short — Sprint 12 ends + # tomorrow, which is what makes "close it and roll the rest over" natural. + past_end_active = (today + timedelta(days=1 if leave_past_open else 7)).isoformat() + # Sprint 13: genuinely active at seed time (start ≤ today ≤ end). + current_start = (today - timedelta(days=3)).isoformat() + current_end = (today + timedelta(days=10)).isoformat() + + # 1) Create Sprint 12 still active (items can be added). + past = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=CYCLE_PAST, + start_date=past_start, + end_date=past_end_active, + owned_by=me_id, + project_id=str(project_id), + ), + ) + # Sprint 13: active window for R4 / W6 transfer target. + current = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=CYCLE_CURRENT, + start_date=current_start, + end_date=current_end, + owned_by=me_id, + project_id=str(project_id), + ), + ) + context["cycles"] = { + CYCLE_PAST: past.id, + CYCLE_CURRENT: current.id, + } + context["cycle_past_id"] = past.id + context["cycle_current_id"] = current.id + + # 2) Add unfinished items to Sprint 12 *before* backdating. + unfinished_ids = [context["items"][title] for title in UNFINISHED_CYCLE_TITLES if title in context["items"]] + if unfinished_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + issue_ids=unfinished_ids, + ) + # R4: items on the active cycle (window still open). + active_ids: list[str] = [] + for title in (PAYMENT_WEBHOOK_TITLE, "Session cookie not rotated after login"): + item_id = context["items"].get(title) + if item_id: + active_ids.append(item_id) + if active_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + issue_ids=active_ids, + ) + overdue_id = context["items"].get("Session cookie not rotated after login") + if overdue_id: + plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=overdue_id, + data=UpdateWorkItem(target_date=(today - timedelta(days=3)).isoformat()), + ) + context["r4_overdue_title"] = "Session cookie not rotated after login" + context["r4_overdue_id"] = overdue_id + context["r4_active_item_ids"] = active_ids + + # 3) Backdate Sprint 12 so it is a completed cycle for R4 semantics — unless the + # task needs to close it itself, in which case it must still be open. + # UpdateCycle.end_date is writable; API allows past end_dates (no "can't backdate" gate + # on the update path — only add_work_items checks end_date < now). + if not leave_past_open: + plane.cycles.update( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + data=UpdateCycle(end_date=past_end_final), + ) + # Final seeded end_date for W6 close assertion (complete_cycle sets end_date=today). + context["cycle_past_seed_end_date"] = past_end_active if leave_past_open else past_end_final + context["cycle_past_open"] = leave_past_open + context["cycle_past_end_date_before_backdate"] = past_end_active diff --git a/evals/seed/intake.py b/evals/seed/intake.py new file mode 100644 index 00000000..f752f971 --- /dev/null +++ b/evals/seed/intake.py @@ -0,0 +1,42 @@ +"""Intake fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest + +INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" +INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" + + +def seed_intake(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + billing = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_BILLING_TITLE, priority="high"), + ), + ) + spam = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_SPAM_TITLE, priority="none"), + ), + ) + # IntakeWorkItem.issue is the work-item id used by triage tools. + context["intake"] = { + "billing": { + "intake_id": billing.id, + "issue_id": getattr(billing, "issue", None) or billing.id, + "title": INTAKE_BILLING_TITLE, + }, + "spam": { + "intake_id": spam.id, + "issue_id": getattr(spam, "issue", None) or spam.id, + "title": INTAKE_SPAM_TITLE, + }, + } diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py new file mode 100644 index 00000000..b68ad84b --- /dev/null +++ b/evals/seed/item_types.py @@ -0,0 +1,79 @@ +"""Work item type fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.work_item_types import CreateWorkItemType + +from .projects import is_plan_gate + + +def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Create or resolve a 'Bug' work item type. + + Genuine plan-gate responses set bug_type=None + skip reason; all other failures raise. + Workspace feature probe uses the real key `is_work_item_types_enabled` (F10). + """ + project_id = context["project_id"] + target = "Bug" + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional work_item_types key alone. + workspace_owns = bool(dump.get("is_work_item_types_enabled")) + + if workspace_owns: + existing = next( + ( + item_type + for item_type in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) + if (item_type.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.workspace_work_item_types.create( + workspace_slug=workspace_slug, data=CreateWorkItemType(name=target) + ) + created = True + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_ids=[existing.id], + ) + context["bug_type"] = {"id": existing.id, "name": target} + context["bug_type_created"] = created + context["bug_type_workspace_level"] = True + if created: + context["workspace_objects"].append({"kind": "work_item_type", "id": existing.id}) + return + + # Per-project types. Project features expose no work-item-type toggle — do not PATCH. + existing = next( + ( + item_type + for item_type in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) + if (item_type.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.work_item_types.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItemType(name=target), + ) + created = True + context["bug_type"] = {"id": existing.id, "name": target} + context["bug_type_created"] = created + context["bug_type_workspace_level"] = False + except Exception as exc: + if is_plan_gate(exc): + context["bug_type"] = None + context["bug_type_skip_reason"] = f"bug_type plan-gated: {exc}" + return + raise diff --git a/evals/seed/labels.py b/evals/seed/labels.py new file mode 100644 index 00000000..410744fa --- /dev/null +++ b/evals/seed/labels.py @@ -0,0 +1,20 @@ +"""Label fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.labels import CreateLabel + +LABEL_NAMES = ("auth", "triage", "perf") + + +def seed_labels(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + for name in LABEL_NAMES: + label = plane.labels.create( + workspace_slug=workspace_slug, + project_id=context["project_id"], + data=CreateLabel(name=name), + ) + context["labels"][name] = label.id diff --git a/evals/seed/modules.py b/evals/seed/modules.py new file mode 100644 index 00000000..860de3b1 --- /dev/null +++ b/evals/seed/modules.py @@ -0,0 +1,61 @@ +"""Fixtures for the Plane Module object, not for Python modules.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.modules import CreateModule +from plane.models.work_items import CreateWorkItem, UpdateWorkItem + +from .work_items import find_completed_state, list_states + +MODULE_NAME = "Checkout revamp" +MODULE_COMPLETED_TITLES = ( + "Module done: cart totals", + "Module done: tax lines", + "Module done: shipping quote", +) + + +def seed_module(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + states = list_states(plane, workspace_slug, project_id) + done = find_completed_state(states) + if done is None: + raise RuntimeError("seed module: no completed-group state to place module items") + + module = plane.modules.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateModule(name=MODULE_NAME, status="in-progress"), + ) + context["module"] = {"id": module.id, "name": MODULE_NAME} + completed_ids: list[str] = [] + for title in MODULE_COMPLETED_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(name=title, priority="medium", state=str(done.id)), # type: ignore[arg-type] + ) + # Force completed state if create ignored it. + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(done.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(done.id)), + ) + completed_ids.append(item.id) + context["items"][title] = item.id + context["item_ids"].append(item.id) + plane.modules.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + module_id=module.id, + issue_ids=completed_ids, + ) + context["module_completed_ids"] = completed_ids + context["module_completed_state_id"] = done.id diff --git a/evals/seed/plan.py b/evals/seed/plan.py new file mode 100644 index 00000000..007b0bec --- /dev/null +++ b/evals/seed/plan.py @@ -0,0 +1,63 @@ +"""Human-readable plans for evaluation fixture creation.""" + +from __future__ import annotations + +from .customers import CUSTOMER_NAME, CUSTOMER_REQUEST_NAME +from .cycles import CYCLE_CURRENT, CYCLE_PAST +from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE +from .labels import LABEL_NAMES +from .modules import MODULE_COMPLETED_TITLES, MODULE_NAME +from .releases import RELEASE_NAME +from .work_items import ( + CHECKOUT_TIMEOUT_TITLE, + DUE_THIS_WEEK_TITLES, + PAYMENT_WEBHOOK_TITLE, + WORK_ITEM_FIXTURES, +) + + +def seed_plan(needs: set[str]) -> list[str]: + """Human-readable seed plan for --dry-run (no network).""" + lines = [ + "project: EVAL {run8} (identifier EV{XXXX})", + ] + if "items" in needs: + lines.append(f"items: {len(WORK_ITEM_FIXTURES)} work items (exactly 4 urgent open)") + lines.append(f" - {PAYMENT_WEBHOOK_TITLE!r} (urgent, non-default started-group state) # R1 target") + lines.append(f" - {len(DUE_THIS_WEEK_TITLES)} assigned-to-me with due this week # R3") + lines.append(f" - comments on {CHECKOUT_TIMEOUT_TITLE!r} # R5 discussion") + if "activity_feed" in needs: + lines.append( + f"activity_feed: gate that activities exist for {CHECKOUT_TIMEOUT_TITLE!r} " + "(TaskSkipped env:no-activity-worker if empty) # L2" + ) + if "labels" in needs: + lines.append(f"labels: {', '.join(LABEL_NAMES)}") + if "bug_type" in needs: + lines.append( + "bug_type: work item type 'Bug' (genuine plan-gate only → skip dependents; other seed errors raise)" + ) + if "cycles" in needs: + past_state = "ends tomorrow, still OPEN so it can be closed" if "cycles_open_past" in needs else "past-dated" + lines.append(f"cycles: {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); unfinished on past") + if "module" in needs: + lines.append(f"module: {MODULE_NAME!r} with {len(MODULE_COMPLETED_TITLES)} completed items") + if "intake" in needs: + lines.append(f"intake: billing {INTAKE_BILLING_TITLE!r} + spam {INTAKE_SPAM_TITLE!r}") + if "customer" in needs: + lines.append(f"customer: {CUSTOMER_NAME!r} + request {CUSTOMER_REQUEST_NAME!r}") + if "release" in needs: + lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") + if "second_project" in needs: + lines.append("second_project: EVAL {run8} B with more open Bug-typed items than main (R6)") + if "leave_cycles_worklogs_off" in needs: + lines.append( + "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " + "(agent enables; teardown re-enables customers=True for later C1)" + ) + else: + lines.append( + "workspace_features: customers=True " + "(is_customer_enabled; NOT work_item_types — leaves S1/S3 type mode alone)" + ) + return lines diff --git a/evals/seed/projects.py b/evals/seed/projects.py new file mode 100644 index 00000000..cc5a18d1 --- /dev/null +++ b/evals/seed/projects.py @@ -0,0 +1,245 @@ +"""Project creation and feature setup for evaluation fixtures.""" + +from __future__ import annotations + +import secrets +from typing import Any + +from plane import PlaneClient +from plane.errors.errors import HttpError +from plane.models.projects import CreateProject, ProjectFeature, UpdateProject +from plane.models.work_items import CreateWorkItem +from plane.models.workspaces import WorkspaceFeature + +# Soft-deleted projects reserve identifiers; create may 409 — retry with a new suffix. +PROJECT_CREATE_ATTEMPT_LIMIT = 3 + +MAIN_PROJECT_BUG_TITLES = ("Main bug alpha", "Main bug beta") +SECOND_PROJECT_BUG_TITLES = ( + "Second bug one", + "Second bug two", + "Second bug three", + "Second bug four", +) + + +def is_plan_gate(exc: BaseException) -> bool: + """True only for genuine plan/subscription feature gates — not generic API failures.""" + if not isinstance(exc, HttpError): + return False + if exc.status_code in (402, 403): + return True + blob = f"{exc} {exc.response!s}".lower() + keywords = ("plan", "subscription", "upgrade", "not available on your", "feature is not enabled") + return any(keyword in blob for keyword in keywords) + + +def is_identifier_collision(exc: BaseException) -> bool: + """True when project create failed because the identifier is already taken. + + Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare + ``identifier`` mention (validation shape errors) must not trigger retry. + """ + if not isinstance(exc, HttpError): + return False + if exc.status_code not in (400, 409): + return False + blob = f"{exc} {exc.response!s}".lower() + return any(keyword in blob for keyword in ("already", "exists", "taken")) + + +def create_project_with_identifier_retry( + plane: PlaneClient, + workspace_slug: str, + *, + name: str, + identifier_prefix: str, + initial_suffix: str, +) -> Any: + """Create a project, regenerating the identifier suffix on soft-delete collisions. + + Plane soft-deletes reserve identifiers; a 409 (or identifier-in-message error) + triggers a new random 4-char hex suffix. At most three attempts are made, + then the last collision error is raised again. + """ + suffix = (initial_suffix or "")[:4].upper() + if len(suffix) < 4: + suffix = (suffix + secrets.token_hex(2).upper())[:4] + last_exc: BaseException | None = None + for attempt in range(PROJECT_CREATE_ATTEMPT_LIMIT): + if attempt > 0: + suffix = secrets.token_hex(2).upper() # 4 hex chars + identifier = f"{identifier_prefix}{suffix}" + try: + return plane.projects.create( + workspace_slug=workspace_slug, + data=CreateProject(name=name, identifier=identifier), + ) + except Exception as exc: + if is_identifier_collision(exc): + last_exc = exc + continue + raise + if last_exc is None: + raise RuntimeError( + f"project create failed after {PROJECT_CREATE_ATTEMPT_LIMIT} identifier retries " + f"(prefix={identifier_prefix!r}) with no captured exception" + ) + raise last_exc + + +def enable_workspace_features( + plane: PlaneClient, + workspace_slug: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> None: + """Enable workspace-level feature toggles that task preconditions need. + + Gate (plane-ee): create-customer 403 when + ``check_workspace_feature(slug, IS_CUSTOMER_ENABLED)`` is false — DB column + ``WorkspaceFeature.is_customer_enabled``. Legacy/SDK flips it via + ``workspaces.update_features`` / ``WorkspaceFeature(customers=True)`` + (API serializer maps ``customers`` → ``is_customer_enabled``). + + Deliberately does **not** set ``work_item_types``: that flips + workspace-vs-project type ownership and would change S1/S3 seed mode. + + ``exclude`` may contain ``customers`` (S5 leaves it off for the agent to enable). + """ + skip = set(exclude or ()) + data: dict[str, bool] = {} + if "customers" not in skip: + data["customers"] = True + if not data: + return + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(**data), + ) + + +def enable_project_features( + plane: PlaneClient, + workspace_slug: str, + project_id: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> None: + """Enable per-project feature gates that fresh projects ship with disabled. + + Two SDK calls (harmless if already on): + + 1. ``projects.update`` / ``UpdateProject`` — view columns API gates read: + ``cycle_view``, ``module_view``, ``intake_view``, ``page_view``, + ``is_time_tracking_enabled`` (worklog 404 when false). + 2. ``projects.update_features`` / ``ProjectFeature`` — capability flags + that the server maps onto the same view columns for cycles/modules/… . + + ``exclude`` is a set of feature keys to leave disabled (for S5): + ``cycles``, ``modules``, ``intakes``, ``pages``, ``worklogs``. + Default: enable all (other catalog tasks need them). + """ + skip = set(exclude or ()) + + update_values: dict[str, bool] = {} + if "cycles" not in skip: + update_values["cycle_view"] = True + if "modules" not in skip: + update_values["module_view"] = True + if "intakes" not in skip: + update_values["intake_view"] = True + if "pages" not in skip: + update_values["page_view"] = True + if "worklogs" not in skip: + update_values["is_time_tracking_enabled"] = True + if update_values: + plane.projects.update( + workspace_slug=workspace_slug, + project_id=project_id, + data=UpdateProject(**update_values), + ) + + feature_values: dict[str, bool] = {} + if "cycles" not in skip: + feature_values["cycles"] = True + if "modules" not in skip: + feature_values["modules"] = True + if "intakes" not in skip: + feature_values["intakes"] = True + if "pages" not in skip: + feature_values["pages"] = True + if feature_values: + plane.projects.update_features( + workspace_slug=workspace_slug, + project_id=project_id, + data=ProjectFeature(**feature_values), + ) + + +def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Seed a second project with more open Bug items than the main project.""" + from .item_types import seed_item_type + + run_prefix = context["run8"] + name = f"EVAL {run_prefix} B" + project = create_project_with_identifier_retry( + plane, + workspace_slug, + name=name, + identifier_prefix="EB", + initial_suffix=run_prefix[:4].upper(), + ) + context["second_project_id"] = project.id + context["second_project_name"] = name + context["second_project_identifier"] = getattr(project, "identifier", None) + # Track for teardown (project delete covers it; still record). + context["second_project_ids"] = [project.id] + enable_project_features(plane, workspace_slug, project.id) + + # Ensure Bug type exists on both projects. + if not context.get("bug_type"): + seed_item_type(plane, workspace_slug, context) + bug = context.get("bug_type") or {} + bug_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_id: + raise RuntimeError("seed second_project: bug_type required for R6 bug counts") + + # Import workspace-level type into second project when needed. + if context.get("bug_type_workspace_level"): + try: + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project.id, + work_item_type_ids=[bug_id], + ) + except Exception as exc: + if not is_plan_gate(exc): + # May already be imported. + if not (isinstance(exc, HttpError) and exc.status_code in (400, 409)): + raise + + main_id = context["project_id"] + # Main project: fewer bugs + main_bug_ids: list[str] = [] + for title in MAIN_PROJECT_BUG_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=main_id, + data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + ) + main_bug_ids.append(item.id) + context["items"][title] = item.id + context["item_ids"].append(item.id) + # Second project: more bugs + second_bug_ids: list[str] = [] + for title in SECOND_PROJECT_BUG_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project.id, + data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + ) + second_bug_ids.append(item.id) + context["r6_main_bug_count"] = len(main_bug_ids) + context["r6_second_bug_count"] = len(second_bug_ids) + context["r6_more_bugs_project"] = name # second project has more diff --git a/evals/seed/releases.py b/evals/seed/releases.py new file mode 100644 index 00000000..9dd1f4f0 --- /dev/null +++ b/evals/seed/releases.py @@ -0,0 +1,34 @@ +"""Release fixtures for evaluation workspaces.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.releases import CreateRelease, UpdateReleaseChangelog + +RELEASE_NAME = "1.2.0" +RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +EVALUATION_RELEASE_TAG_VERSION = "eval-rc1" + + +def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + release = plane.releases.create( + workspace_slug=workspace_slug, + data=CreateRelease(name=RELEASE_NAME), + ) + context["release"] = {"id": release.id, "name": RELEASE_NAME} + context["workspace_objects"].append({"kind": "release", "id": release.id}) + # Single changelog body; DESIGN's "2 entries" are encoded as plain-text bullets. + try: + plane.releases.changelog.update( + workspace_slug=workspace_slug, + release_id=release.id, + data=UpdateReleaseChangelog( + description_html=f"

{RELEASE_CHANGELOG_TEXT}

", + ), + ) + except Exception as exc: + # Non-fatal for seed if changelog endpoint is flaky; C2 verifier still checks release name. + print(f"seed warning: release changelog update failed: {exc}") + context["release_changelog_text"] = RELEASE_CHANGELOG_TEXT diff --git a/evals/seed/remove.py b/evals/seed/remove.py new file mode 100644 index 00000000..02e16aed --- /dev/null +++ b/evals/seed/remove.py @@ -0,0 +1,209 @@ +"""Fixture removal for evaluation runs.""" + +from __future__ import annotations + +import os +from typing import Any + +from plane import PlaneClient +from plane.errors.errors import HttpError +from plane.models.workspaces import WorkspaceFeature + +from .customers import CUSTOMER_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME +from .releases import EVALUATION_RELEASE_TAG_VERSION + + +def _remove_severity_property(plane: PlaneClient, context: dict[str, Any]) -> None: + """Delete Severity properties attached to the seeded Bug type (avoids multi-rep pollution).""" + bug = context.get("bug_type") + if not bug: + return + bug_type_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_type_id: + return + workspace_slug = context.get("workspace_slug") or "" + project_id = context.get("project_id") + + properties: list[Any] = [] + try: + if project_id: + properties = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + except HttpError as exc: + if exc.status_code not in (404, 405): + print(f"teardown warning: list Severity props failed: {exc}") + return + except Exception as exc: + print(f"teardown warning: list Severity props failed: {exc}") + return + + for work_item_property in properties: + display = ( + getattr(work_item_property, "display_name", None) or getattr(work_item_property, "name", None) or "" + ).strip() + if display.lower() != "severity": + continue + try: + if project_id: + plane.work_item_properties.delete( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + work_item_property_id=work_item_property.id, + ) + context.setdefault("workspace_objects", []) # no-op anchor + except Exception as exc: + print(f"teardown warning: failed to delete Severity property {work_item_property.id}: {exc}") + + +def _remove_incident_type(plane: PlaneClient, context: dict[str, Any]) -> None: + """Best-effort cleanup of agent-created Incident type (S3 multi-rep pollution).""" + workspace_slug = context.get("workspace_slug") or "" + project_id = context.get("project_id") + try: + if context.get("bug_type_workspace_level"): + for item_type in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []: + if (item_type.name or "").strip().casefold() == "incident": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=item_type.id) + elif project_id: + for item_type in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []: + if (item_type.name or "").strip().casefold() == "incident": + plane.work_item_types.delete( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_id=item_type.id, + ) + except Exception as exc: + print(f"teardown warning: Incident type cleanup failed: {exc}") + + +def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: + """Delete the project and any workspace-scoped objects we created.""" + if not ctx: + return + workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + project_id = ctx.get("project_id") + + # S5 left customers off (or agent enabled them): re-enable for subsequent task-reps + # on the shared eval workspace. We always set customers=True — we do not restore a + # prior false (S5's job is to leave the workspace usable for C1). + if ctx.get("s5_left_customers_off"): + try: + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(customers=True), + ) + except Exception as exc: + print(f"teardown warning: re-enable workspace customers failed: {exc}") + + # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). + try: + _remove_severity_property(plane, ctx) + except Exception as exc: + print(f"teardown warning: Severity cleanup failed: {exc}") + try: + _remove_incident_type(plane, ctx) + except Exception as exc: + print(f"teardown warning: Incident cleanup failed: {exc}") + + # Best-effort: agent-created Acme Corp customers (C1) that never hit workspace_objects. + try: + page = plane.customers.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + for customer in rows or []: + if (customer.name or "").strip().casefold() in (CUSTOMER_NAME.casefold(), "acme"): + # Only delete if we seeded or created during this run (tracked or name match + run). + tracked = { + obj.get("id") for obj in (ctx.get("workspace_objects") or []) if obj.get("kind") == "customer" + } + if str(customer.id) in tracked or ctx.get("customer") is None: + # Avoid deleting long-lived Acme if we pre-seeded and tracked it — still delete tracked. + if str(customer.id) in tracked or not ctx.get("customer"): + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": customer.id}) + except Exception as exc: + print(f"teardown warning: customer scan failed: {exc}") + + # Workspace-scoped cleanup first (survive project deletion). + seen_workspace_objects: set[str] = set() + for obj in ctx.get("workspace_objects") or []: + kind = obj.get("kind") + object_id = obj.get("id") + if not object_id: + continue + key = f"{kind}:{object_id}" + if key in seen_workspace_objects: + continue + seen_workspace_objects.add(key) + try: + if kind == "work_item_type": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=object_id) + elif kind == "work_item_property": + plane.workspace_work_item_properties.delete(workspace_slug=workspace_slug, property_id=object_id) + elif kind == "customer": + plane.customers.delete(workspace_slug=workspace_slug, customer_id=object_id) + elif kind == "release": + plane.releases.delete(workspace_slug=workspace_slug, release_id=object_id) + elif kind == "release_tag": + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=object_id) + elif kind == "customer_property": + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=object_id) + except Exception as exc: + print(f"teardown warning: failed to delete workspace {kind} {object_id}: {exc}") + + # Sweep by well-known WS3 names in case tracking missed an agent-created row. + try: + page = plane.releases.tags.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + for tag in rows or []: + if (getattr(tag, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION: + tag_id = getattr(tag, "id", None) + if tag_id and f"release_tag:{tag_id}" not in seen_workspace_objects: + try: + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tag_id) + except Exception as exc: + print(f"teardown warning: sweep release tag {tag_id}: {exc}") + except Exception as exc: + print(f"teardown warning: sweep release tags failed: {exc}") + try: + page = plane.customers.properties.list(workspace_slug=workspace_slug) + rows = page.results if hasattr(page, "results") else page + target = EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + for customer_property in rows or []: + display = ( + getattr(customer_property, "display_name", None) or getattr(customer_property, "name", None) or "" + ).strip() + if display.casefold() == target: + property_id = getattr(customer_property, "id", None) + if property_id and f"customer_property:{property_id}" not in seen_workspace_objects: + try: + plane.customers.properties.delete( + workspace_slug=workspace_slug, + property_id=property_id, + ) + except Exception as exc: + print(f"teardown warning: sweep customer property {property_id}: {exc}") + except Exception as exc: + print(f"teardown warning: sweep customer properties failed: {exc}") + + # Second project before main (no dependency either way, but be thorough). + for second_project_id in ctx.get("second_project_ids") or []: + if not second_project_id or second_project_id == project_id: + continue + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=second_project_id) + except Exception as exc: + print(f"teardown warning: failed to delete second project {second_project_id}: {exc}") + + if project_id: + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + name = ctx.get("project_name", project_id) + print(f"teardown warning: failed to delete project {name!r}: {exc}") + print(f"orphaned project: {name}") diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py new file mode 100644 index 00000000..868c9562 --- /dev/null +++ b/evals/seed/work_items.py @@ -0,0 +1,167 @@ +"""Work item fixtures for evaluation projects.""" + +from __future__ import annotations + +from datetime import date, timedelta +from typing import Any + +from plane import PlaneClient +from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem + +# Fixed fixture titles for the `items` group. Exactly 4 urgent; the rest medium/high/low. +# "Payment webhook drops retries" is the R1 target (urgent, non-default state). +WORK_ITEM_FIXTURES: list[tuple[str, str]] = [ + ("Payment webhook drops retries", "urgent"), + ("Checkout times out on 3DS challenge", "urgent"), + ("Session cookie not rotated after login", "urgent"), + ("Inventory count goes negative under load", "urgent"), + ("Search results ignore archived projects", "high"), + ("CSV export truncates multi-byte chars", "high"), + ("Webhook secret rotation docs missing", "medium"), + ("Dark mode contrast fails WCAG AA", "medium"), + ("Onboarding email template stale", "medium"), + ("Sidebar collapse flickers on resize", "low"), + ("Tooltip clipped inside modal dialog", "low"), + ("Footer year still says 2024", "none"), +] + +PAYMENT_WEBHOOK_TITLE = WORK_ITEM_FIXTURES[0][0] +# R5 discussion target + distinctive comment phrases (word-boundary matched at verify). +CHECKOUT_TIMEOUT_TITLE = "Checkout times out on 3DS challenge" +CHECKOUT_COMMENT_PHRASES = ( + "stripe callback race", + "retry budget exhausted", +) +# W2 / W3 / W8 targets +SIDEBAR_TITLE = "Sidebar collapse flickers on resize" +DARK_MODE_TITLE = "Dark mode contrast fails WCAG AA" +# W7 relation pair + reference URL +BLOCKING_SOURCE_TITLE = "Search results ignore archived projects" +BLOCKING_TARGET_TITLE = "CSV export truncates multi-byte chars" +BLOCKING_REFERENCE_ADDRESS = "https://example.com/eval/runbook-w7" +# R3: assignees + due this week (seeded count stored in ctx) +DUE_THIS_WEEK_TITLES = ( + "Webhook secret rotation docs missing", + "Onboarding email template stale", +) +# W6 unfinished items in Sprint 12 +UNFINISHED_CYCLE_TITLES = ( + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", +) + + +def list_states(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + return list(page.results or []) + + +def find_completed_state(states: list[Any]) -> Any | None: + completed = [state for state in states if getattr(state, "group", None) == "completed"] + if not completed: + return None + # Prefer a non-default completed state named Done if present. + for state in completed: + if (state.name or "").strip().casefold() == "done": + return state + return completed[0] + + +def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + states = list_states(plane, workspace_slug, project_id) + context["state_names"] = sorted({(state.name or "").strip() for state in states if (state.name or "").strip()}) + + # Prefer a non-default started-group state so R1 cannot be passed by guessing the default. + started = [ + state for state in states if getattr(state, "group", None) == "started" and not getattr(state, "default", False) + ] + if not started: + started = [state for state in states if getattr(state, "group", None) == "started"] + if not started: + raise RuntimeError( + "seed items: no started-group state available to place the R1 target; " + f"states={[(state.name, state.group, state.default) for state in states]}" + ) + r1_state = started[0] + context["r1_state_name"] = r1_state.name + context["r1_state_id"] = r1_state.id + + me = plane.users.get_me() + me_id = str(me.id) + context["me_id"] = me_id + # Due dates must stay inside the current ISO week (Mon–Sun). + # today+2d alone escapes the week on Sat/Sun — clamp to this week's Sunday. + today = date.today() + days_to_week_end = 6 - today.weekday() # Mon=0 … Sun=6 + due_this_week = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)).isoformat() + context["r3_due_date"] = due_this_week + + urgent_count = 0 + for title, priority in WORK_ITEM_FIXTURES: + data_kwargs: dict[str, Any] = {"name": title, "priority": priority} + if title == PAYMENT_WEBHOOK_TITLE: + data_kwargs["state"] = str(r1_state.id) + if title in DUE_THIS_WEEK_TITLES: + data_kwargs["assignees"] = [me_id] + data_kwargs["target_date"] = due_this_week + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(**data_kwargs), # type: ignore[arg-type] + ) + # Some APIs ignore state on create; force via update if needed. + if title == PAYMENT_WEBHOOK_TITLE: + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(r1_state.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(r1_state.id)), + ) + context["items"][title] = item.id + context["item_ids"].append(item.id) + sequence = getattr(item, "sequence_id", None) + if sequence is not None and context.get("project_identifier"): + context["item_identifiers"][title] = f"{context['project_identifier']}-{sequence}" + if priority == "urgent": + urgent_count += 1 + assert urgent_count == 4, f"fixture invariant: expected 4 urgent items, got {urgent_count}" + + # R5: seed discussion comments on the known item. + target_id = context["items"].get(CHECKOUT_TIMEOUT_TITLE) + if target_id: + for phrase in CHECKOUT_COMMENT_PHRASES: + plane.work_items.comments.create( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + data=CreateWorkItemComment(comment_html=f"

{phrase}

"), + ) + + +def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Skip L2 when comments never materialize as activities (no activity worker). + + Raises :class:`evals.tasks.TaskSkipped` with reason ``env:no-activity-worker`` + so the harness records a skip, not a task failure. + """ + from evals.tasks import TaskSkipped + + project_id = context.get("project_id") + work_item_id = (context.get("items") or {}).get(CHECKOUT_TIMEOUT_TITLE) + if not project_id or not work_item_id: + raise TaskSkipped("env:no-activity-worker") + try: + page = plane.work_items.activities.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + except Exception as exc: + raise TaskSkipped(f"env:no-activity-worker ({type(exc).__name__}: {exc})") from exc + rows = page.results if hasattr(page, "results") else page + if len(list(rows or [])) < 1: + raise TaskSkipped("env:no-activity-worker") From 38e19918952074e7a31ab68ae9a8da51acf75a8a Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 14:45:06 +0530 Subject: [PATCH 14/93] Split the report by job and read rows only through the schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report.py was 846 lines doing four unrelated jobs — statistics, row loading, summarising, and rendering — and it grew by 200 lines in a single session, which made it the fastest-degrading file in the harness. Each job now has a module: statistics, load, summary, table, compare, with the CLI in __main__ so `python -m evals.report` is unchanged. Statistics stays inside the package rather than moving to the root, because nothing else imports Wilson or the sign test; it graduates the day a second package does. The statistics themselves moved verbatim — a cleanup that quietly altered how an interval is computed would invalidate every past comparison while reading as cosmetic. Rows are now read only through TaskResult, so row-shape knowledge lives solely in results.py. What remains untyped is one level up: summarize() still returns a bare dict whose _meta keys the renderers reach into, which is the same problem worth fixing next. Verified by output equivalence rather than test count, on all three paths against the previous commit: single-rep tables, multi-rep aggregation on a file with deliberately unstable tasks, and the A/B comparison. All byte-identical. Co-Authored-By: Claude Fable 5 --- evals/report.py | 846 --------------------------------- evals/report/__init__.py | 63 +++ evals/report/__main__.py | 13 + evals/report/command.py | 105 ++++ evals/report/compare.py | 122 +++++ evals/report/load.py | 90 ++++ evals/report/statistics.py | 65 +++ evals/report/summary.py | 168 +++++++ evals/report/table.py | 379 +++++++++++++++ evals/results.py | 4 + tests/test_evals_report_ops.py | 3 + 11 files changed, 1012 insertions(+), 846 deletions(-) delete mode 100644 evals/report.py create mode 100644 evals/report/__init__.py create mode 100644 evals/report/__main__.py create mode 100644 evals/report/command.py create mode 100644 evals/report/compare.py create mode 100644 evals/report/load.py create mode 100644 evals/report/statistics.py create mode 100644 evals/report/summary.py create mode 100644 evals/report/table.py diff --git a/evals/report.py b/evals/report.py deleted file mode 100644 index e664f108..00000000 --- a/evals/report.py +++ /dev/null @@ -1,846 +0,0 @@ -"""Summary table, A/B delta, and multi-surface tables for eval JSONL results. - -Usage: - python -m evals.report evals/results/A.jsonl - python -m evals.report A.jsonl B.jsonl # A/B delta (sign test + Wilson) - python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface - python -m evals.report --table --markdown f1.jsonl f2.jsonl -""" - -from __future__ import annotations - -import argparse -import json -import math -import sys -from collections import defaultdict -from pathlib import Path -from typing import Any, Literal - -from evals.results import TaskResult -from evals.tasks import TASKS_BY_ID - -DedupeMode = Literal["latest", "none"] -ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] -ResultRow = TaskResult | dict[str, Any] - - -def _task_result(row: ResultRow) -> TaskResult: - return row if isinstance(row, TaskResult) else TaskResult.from_row(row) - - -def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: - """95% Wilson score interval for a binomial proportion.""" - if n <= 0: - return (0.0, 0.0) - p = k / n - z2 = z * z - denom = 1.0 + z2 / n - centre = p + z2 / (2.0 * n) - margin = z * math.sqrt((p * (1.0 - p) + z2 / (4.0 * n)) / n) - lo = max(0.0, (centre - margin) / denom) - hi = min(1.0, (centre + margin) / denom) - return (lo, hi) - - -def sign_test_pvalue(deltas: list[float]) -> float | None: - """Two-sided exact binomial sign test on non-zero paired deltas. - - H0: P(delta > 0) = 1/2. Zero deltas are dropped. Returns None when no - non-zero pairs remain. Uses ``math.comb`` only (no scipy). - """ - nonzero = [d for d in deltas if d != 0] - n = len(nonzero) - if n == 0: - return None - k = sum(1 for d in nonzero if d > 0) - total = 2**n - # Two-sided: 2 * min(left cdf, right survival), capped at 1. - left = sum(math.comb(n, i) for i in range(0, k + 1)) / total - right = sum(math.comb(n, i) for i in range(k, n + 1)) / total - return min(1.0, 2.0 * min(left, right)) - - -def _median(xs: list[float]) -> float | None: - if not xs: - return None - s = sorted(xs) - m = len(s) // 2 - if len(s) % 2: - return float(s[m]) - return (s[m - 1] + s[m]) / 2.0 - - -def _percentile(xs: list[float], p: float) -> float | None: - if not xs: - return None - s = sorted(xs) - if len(s) == 1: - return float(s[0]) - k = (len(s) - 1) * p - f = math.floor(k) - c = math.ceil(k) - if f == c: - return float(s[int(k)]) - return float(s[f] + (s[c] - s[f]) * (k - f)) - - -def _iqr(xs: list[float]) -> tuple[float | None, float | None, float | None]: - return (_percentile(xs, 0.25), _median(xs), _percentile(xs, 0.75)) - - -def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: - """Classify token counts without treating unmarked legacy data as measured.""" - labels: set[str] = set() - for raw_row in rows: - row = _task_result(raw_row) - for call in row.calls: - if call.result_tokens is None: - continue - estimated = ( - call.result_tokens_estimated - if call.result_tokens_estimated is not None - else row.result_tokens_estimated - ) - if estimated is True: - labels.add("estimated") - elif estimated is False: - labels.add("measured") - else: - labels.add("unlabeled") - if not labels: - return "unavailable" - if labels == {"estimated"}: - return "estimated" - if labels == {"measured"}: - return "measured" - if "unlabeled" in labels: - return "unlabeled" - return "mixed" - - -def is_meta_row(row: ResultRow) -> bool: - """True for run-header meta lines (or any row without a task_id).""" - if isinstance(row, TaskResult): - return not row.task_id - if row.get("row_type") == "meta": - return True - return row.get("task_id") is None - - -def is_infra_error_row(row: ResultRow) -> bool: - """True when a row failed for infrastructure reasons (seed/cli/api), not task verify. - - Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, - ``infra_api``, ``infra_sdk``, …) is excluded from success-rate denominators. - """ - ec = _task_result(row).error_class - return isinstance(ec, str) and ec.startswith("infra_") - - -def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: - """Keep only the last row per (task_id, rep, surface); preserve key insertion order.""" - latest: dict[tuple[str, int, str], TaskResult] = {} - order: list[tuple[str, int, str]] = [] - for raw_row in rows: - row = _task_result(raw_row) - key = (row.task_id, row.rep, row.surface) - if key not in latest: - order.append(key) - latest[key] = row - return [latest[k] for k in order] - - -def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: - """Load JSONL data rows (skip meta / missing task_id). - - Default ``dedupe="latest"`` keeps the last row per (task_id, rep, surface) - so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. - """ - rows: list[TaskResult] = [] - with path.open(encoding="utf-8") as fh: - for line_no, line in enumerate(fh, start=1): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError as exc: - print( - f"warning: {path}:{line_no}: skipping invalid JSON ({exc})", - file=sys.stderr, - ) - continue - if not isinstance(row, dict) or is_meta_row(row): - continue - rows.append(TaskResult.from_row(row)) - if dedupe == "latest": - return dedupe_rows_latest(rows) - # Forensics: warn on duplicates but keep all. - seen_keys: set[tuple[str, int, str]] = set() - for r in rows: - key = (r.task_id, r.rep, r.surface) - if key in seen_keys: - print( - f"warning: {path}: duplicate (task_id, rep, surface)={key} " - f"(--no-dedupe keeps all rows; bare --out reuse double-counts)", - file=sys.stderr, - ) - else: - seen_keys.add(key) - return rows - - -def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: - """Aggregate per-task metrics. - - Rows with ``error_class`` starting ``infra_`` are excluded from success-rate - denominators and counted separately as ``infra_errors`` (total on the returned - dict under the special key ``_meta``). Other non-null ``error`` rows remain - harness errors (excluded from success, counted in ``harness_err``). - """ - by_task: dict[str, list[TaskResult]] = defaultdict(list) - harness_err_by_task: dict[str, int] = defaultdict(int) - infra_err_by_task: dict[str, int] = defaultdict(int) - reps_by_task: dict[str, set[int]] = defaultdict(set) - infra_errors = 0 - for raw_row in rows: - r = _task_result(raw_row) - if is_meta_row(r): - continue - tid = r.task_id - reps_by_task[tid].add(r.rep) - if is_infra_error_row(r): - infra_errors += 1 - infra_err_by_task[tid] += 1 - continue # infra seed/cli — excluded from success aggregates - if r.error: - harness_err_by_task[tid] += 1 - continue # harness/API errors excluded from success/medians (F4) - if r.skipped: - continue # skipped rows are excluded from success denominators - by_task[tid].append(r) - - # Include tasks that only had harness/infra errors so columns stay visible. - all_task_ids = sorted(set(by_task) | set(harness_err_by_task) | set(infra_err_by_task)) - - out: dict[str, dict[str, Any]] = {} - total_k = 0 - total_n = 0 - for task_id in all_task_ids: - trs = by_task.get(task_id, []) - n = len(trs) - k = sum(1 for r in trs if r.success) - unstable = n > 1 and 0 < k < n - total_k += k - total_n += n - lo, hi = wilson_interval(k, n) if n else (0.0, 0.0) - calls = [float(r.num_calls) for r in trs] - q1, med_calls, q3 = _iqr(calls) - min_calls = min(calls) if calls else None - max_calls = max(calls) if calls else None - optimal = TASKS_BY_ID.get(task_id, {}).get("optimal_calls") - total_calls = 0 - mispick = 0 - errored = 0 - result_tokens: list[float] = [] - for r in trs: - for c in r.calls: - total_calls += 1 - if c.classification in ("alternate", "out_of_set"): - mispick += 1 - if c.is_error: - errored += 1 - if c.result_tokens is not None: - result_tokens.append(float(c.result_tokens)) - capped = sum(1 for r in trs if r.hit_max_iterations or r.stop_reason == "max_tokens") - cum_inputs = [float(r.cum_input_tokens or 0) for r in trs] - out[task_id] = { - "n": n, - "k": k, - "success": f"{k}/{n}" if n else "0/0", - "unstable": unstable, - "wilson_lo": lo, - "wilson_hi": hi, - "med_calls": med_calls, - "calls_min": min_calls, - "calls_max": max_calls, - "calls_q1": q1, - "calls_q3": q3, - "optimal_calls": optimal, - "mispick_rate": (mispick / total_calls) if total_calls else 0.0, - "errored_calls": errored, - "capped": capped, - "harness_err": harness_err_by_task.get(task_id, 0), - "infra_err": infra_err_by_task.get(task_id, 0), - "med_result_tokens": _median(result_tokens), - "p95_result_tokens": _percentile(result_tokens, 0.95), - "result_tokens_mode": result_tokens_mode(trs), - "med_cum_input": _median(cum_inputs), - } - agg_lo, agg_hi = wilson_interval(total_k, total_n) if total_n else (0.0, 0.0) - unstable_task_ids = sorted(task_id for task_id, values in out.items() if values.get("unstable")) - out["_meta"] = { - "infra_errors": infra_errors, - "aggregate_k": total_k, - "aggregate_n": total_n, - "aggregate_wilson_lo": agg_lo, - "aggregate_wilson_hi": agg_hi, - "multi_rep": any(len(reps) > 1 for reps in reps_by_task.values()), - "unstable_task_ids": unstable_task_ids, - "unstable_tasks": len(unstable_task_ids), - "result_tokens_mode": result_tokens_mode([r for trs in by_task.values() for r in trs]), - } - return out - - -def _fmt(x: float | None, digits: int = 1) -> str: - if x is None: - return "-" - return f"{x:.{digits}f}" - - -def _result_tokens_marker(mode: str) -> str: - return {"estimated": "~", "mixed": "*", "unlabeled": "?"}.get(mode, "") - - -def _fmt_result_tokens(x: float | None, mode: str) -> str: - value = _fmt(x, 0) - if value == "-": - return value - return f"{_result_tokens_marker(mode)}{value}" - - -def noise_floor_statement(unstable_tasks: int) -> str: - """Describe observed pass/fail variance in task-count comparison units.""" - count = max(0, int(unstable_tasks)) - if count == 0: - return ( - "measured noise floor: 0 tasks flipped at least once; no non-zero " - "run-to-run variance was observed (minimum meaningful difference " - "from observed flips: 1 task)" - ) - noun = "task" if count == 1 else "tasks" - threshold = count + 1 - threshold_noun = "task" if threshold == 1 else "tasks" - return ( - f"measured noise floor: {count} {noun} flipped at least once; surface " - f"differences of {count} {noun} or fewer are within observed run-to-run " - f"variance (minimum meaningful difference: {threshold} {threshold_noun})" - ) - - -def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: - meta = summary.get("_meta") or {} - print(title) - token_mode = str(meta.get("result_tokens_mode") or "unavailable") - if token_mode == "estimated": - print("result-token columns marked ~: entirely estimated from result characters") - elif token_mode == "mixed": - print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") - elif token_mode == "unlabeled": - print("result-token columns marked ?: include legacy values with unknown measurement status") - if meta.get("infra_errors"): - print(f"infra errors: {meta['infra_errors']}") - agg_n = int(meta.get("aggregate_n") or 0) - if agg_n: - agg_k = int(meta.get("aggregate_k") or 0) - alo = float(meta.get("aggregate_wilson_lo") or 0.0) - ahi = float(meta.get("aggregate_wilson_hi") or 0.0) - rate = agg_k / agg_n if agg_n else 0.0 - print(f"aggregate success: {agg_k}/{agg_n} ({rate:.1%}) Wilson95 [{alo:.2f},{ahi:.2f}]") - multi_rep = bool(meta.get("multi_rep")) - if multi_rep: - print(noise_floor_statement(int(meta.get("unstable_tasks") or 0))) - # Multi-rep files keep the repetition-aware layout even when errors leave - # only one completed result in every task's success-rate denominator. - show_var = multi_rep or any(s.get("n", 0) > 1 for tid, s in summary.items() if tid != "_meta") - token_marker = _result_tokens_marker(token_mode) - med_rtok_header = f"med_rtok{token_marker}" - p95_rtok_header = f"p95_rtok{token_marker}" - if show_var: - unstable_header = f"{'unstable':>8} " if multi_rep else "" - header = ( - f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " - f"{unstable_header}" - f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " - f"{'IQR':>11} {'mispick':>8} {'err':>4} " - f"{'capped':>6} {'h_err':>5} {'i_err':>5} {med_rtok_header:>9} {p95_rtok_header:>9} " - f"{'med_cum_in':>10}" - ) - else: - header = ( - f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " - f"{'med_calls':>9} {'opt':>4} {'IQR':>11} {'mispick':>8} {'err':>4} " - f"{'capped':>6} {'h_err':>5} {'i_err':>5} {med_rtok_header:>9} {p95_rtok_header:>9} " - f"{'med_cum_in':>10}" - ) - print(header) - print("-" * len(header)) - for task_id, s in summary.items(): - if task_id == "_meta": - continue - wilson = f"[{s['wilson_lo']:.2f},{s['wilson_hi']:.2f}]" - iqr = f"{_fmt(s['calls_q1'])}-{_fmt(s['calls_q3'])}" - opt = s["optimal_calls"] if s["optimal_calls"] is not None else "-" - task_token_mode = str(s.get("result_tokens_mode") or "unavailable") - if show_var: - unstable = ("YES" if s.get("unstable") else "no") if multi_rep else "" - unstable_cell = f"{unstable:>8} " if multi_rep else "" - print( - f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " - f"{unstable_cell}" - f"{_fmt(s.get('calls_min')):>9} {_fmt(s['med_calls']):>9} {_fmt(s.get('calls_max')):>9} " - f"{opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " - f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " - f"{s.get('infra_err', 0):>5} " - f"{_fmt_result_tokens(s['med_result_tokens'], task_token_mode):>9} " - f"{_fmt_result_tokens(s['p95_result_tokens'], task_token_mode):>9} " - f"{_fmt(s['med_cum_input'], 0):>10}" - ) - else: - print( - f"{task_id:<6} {s['n']:>3} {s['success']:>8} {wilson:>16} " - f"{_fmt(s['med_calls']):>9} {opt!s:>4} {iqr:>11} {s['mispick_rate']:>7.1%} " - f"{s['errored_calls']:>4} {s['capped']:>6} {s['harness_err']:>5} " - f"{s.get('infra_err', 0):>5} " - f"{_fmt_result_tokens(s['med_result_tokens'], task_token_mode):>9} " - f"{_fmt_result_tokens(s['p95_result_tokens'], task_token_mode):>9} " - f"{_fmt(s['med_cum_input'], 0):>10}" - ) - - -# --------------------------------------------------------------------------- -# A/B comparison -# --------------------------------------------------------------------------- - - -def ab_compare( - rows_a: list[ResultRow], - rows_b: list[ResultRow], -) -> dict[str, Any]: - """Compare two result sets: paired call-count deltas + success rates. - - Paired call deltas only include tasks with at least one successful repetition - in both A and B. Calls are the median across successful repetitions; this is - identical to the historical behavior for single-rep files. - """ - sum_a = summarize(rows_a) - sum_b = summarize(rows_b) - - def _success_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: - out: dict[str, list[float]] = defaultdict(list) - for raw_row in rows: - r = _task_result(raw_row) - if is_meta_row(r) or is_infra_error_row(r) or r.error or r.skipped: - continue - if not r.success: - continue - out[r.task_id].append(float(r.num_calls)) - return dict(out) - - sa = _success_calls_by_task(rows_a) - sb = _success_calls_by_task(rows_b) - shared = sorted(set(sa) & set(sb)) - deltas: list[float] = [] - per_task: list[dict[str, Any]] = [] - for tid in shared: - ca = float(_median(sa[tid]) or 0.0) - cb = float(_median(sb[tid]) or 0.0) - d = cb - ca # B − A (negative = B fewer calls = better if lower is better) - deltas.append(d) - per_task.append({"task_id": tid, "calls_a": ca, "calls_b": cb, "delta": d}) - - meta_a = sum_a.get("_meta") or {} - meta_b = sum_b.get("_meta") or {} - return { - "summary_a": sum_a, - "summary_b": sum_b, - "paired_tasks": per_task, - "median_delta": _median(deltas), - "sign_test_p": sign_test_pvalue(deltas), - "n_paired": len(deltas), - "multi_rep": bool(meta_a.get("multi_rep") or meta_b.get("multi_rep")), - "unstable_a": int(meta_a.get("unstable_tasks") or 0), - "unstable_b": int(meta_b.get("unstable_tasks") or 0), - "success_a": { - "k": int(meta_a.get("aggregate_k") or 0), - "n": int(meta_a.get("aggregate_n") or 0), - "wilson": ( - float(meta_a.get("aggregate_wilson_lo") or 0.0), - float(meta_a.get("aggregate_wilson_hi") or 0.0), - ), - }, - "success_b": { - "k": int(meta_b.get("aggregate_k") or 0), - "n": int(meta_b.get("aggregate_n") or 0), - "wilson": ( - float(meta_b.get("aggregate_wilson_lo") or 0.0), - float(meta_b.get("aggregate_wilson_hi") or 0.0), - ), - }, - } - - -def print_ab_report(cmp: dict[str, Any], path_a: Path, path_b: Path) -> None: - print(f"A/B compare: A={path_a} B={path_b}") - sa, sb = cmp["success_a"], cmp["success_b"] - ra = (sa["k"] / sa["n"]) if sa["n"] else 0.0 - rb = (sb["k"] / sb["n"]) if sb["n"] else 0.0 - print(f" success A: {sa['k']}/{sa['n']} ({ra:.1%}) Wilson95 [{sa['wilson'][0]:.2f},{sa['wilson'][1]:.2f}]") - print(f" success B: {sb['k']}/{sb['n']} ({rb:.1%}) Wilson95 [{sb['wilson'][0]:.2f},{sb['wilson'][1]:.2f}]") - print(f" success rate delta (B−A): {rb - ra:+.1%}") - print(f" paired successful tasks: {cmp['n_paired']}") - print(f" median call delta (B−A): {_fmt(cmp['median_delta'])}") - p = cmp["sign_test_p"] - print(f" sign-test p-value (two-sided): {p if p is not None else 'n/a'}") - multi_rep = bool(cmp.get("multi_rep")) - if multi_rep: - print(f" A {noise_floor_statement(int(cmp.get('unstable_a') or 0))}") - print(f" B {noise_floor_statement(int(cmp.get('unstable_b') or 0))}") - if cmp["paired_tasks"]: - print() - print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") - print("-" * 34) - for row in cmp["paired_tasks"]: - if multi_rep: - print(f"{row['task_id']:<6} {_fmt(row['calls_a']):>8} {_fmt(row['calls_b']):>8} {row['delta']:>+8.1f}") - else: - print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") - - -# --------------------------------------------------------------------------- -# Multi-surface table -# --------------------------------------------------------------------------- - - -def _task_sort_key(tid: str) -> tuple[str, int]: - digits = "".join(c for c in tid if c.isdigit()) - return (tid[0] if tid else "", int(digits) if digits else 0) - - -def format_surface_cell(row: ResultRow | None) -> str: - """Cell for multi-surface table: '✅ Nc/Mmp', 'skip', 'ERR', or '—'.""" - if row is None: - return "—" - result = _task_result(row) - if result.skipped: - return "skip" - if result.error or is_infra_error_row(result): - return "ERR" - ok = "✅" if result.success else "❌" - n_calls_s = str(result.num_calls) - if result.classification == "external": - return f"{ok} {n_calls_s}c" - alt = result.alternate_calls - oos = result.out_of_set_calls - # None counters (external nulling) → omit mispick suffix. - if alt is None and oos is None: - return f"{ok} {n_calls_s}c" - mp = int(alt or 0) + int(oos or 0) - if mp: - return f"{ok} {n_calls_s}c/{mp}mp" - return f"{ok} {n_calls_s}c" - - -def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: - """Aggregate distinct repetitions into one task/surface cell.""" - results = [_task_result(row) for row in rows] - completed = [row for row in results if not is_infra_error_row(row) and not row.error and not row.skipped] - if completed: - n = len(completed) - k = sum(1 for row in completed if row.success) - lo, hi = wilson_interval(k, n) - if 0 < k < n: - marker = "⚠ UNSTABLE" - elif k == n: - marker = "✅" - else: - marker = "❌" - calls = [row.num_calls for row in completed] - call_span = f"{min(calls)}c" if min(calls) == max(calls) else f"{min(calls)}-{max(calls)}c" - return f"{marker} {k}/{n} [{lo:.2f},{hi:.2f}] {call_span}" - if any(row.error or is_infra_error_row(row) for row in results): - return "ERR" - if any(row.skipped for row in results): - return "skip" - return "—" - - -def build_multi_surface_table( - file_rows: list[tuple[str, list[ResultRow]]], -) -> dict[str, Any]: - """Build a per-task × per-surface grid from labeled row sets. - - ``file_rows`` is a list of ``(column_label, rows)``. Column labels default - to each file's dominant ``surface`` field when the caller passes that label. - Rows are grouped by task and repetition. Single-rep columns retain the - historical one-cell rendering; multi-rep columns aggregate all repetitions. - """ - columns: list[str] = [] - by_col: dict[str, dict[str, list[TaskResult]]] = {} - multi_rep_by_col: dict[str, bool] = {} - for label, rows in file_rows: - columns.append(label) - col_map: dict[str, list[TaskResult]] = defaultdict(list) - for raw_row in rows: - if is_meta_row(raw_row): - continue - r = _task_result(raw_row) - tid = r.task_id - col_map[tid].append(r) - by_col[label] = dict(col_map) - multi_rep_by_col[label] = any(len({row.rep for row in task_rows}) > 1 for task_rows in col_map.values()) - - multi_rep = any(multi_rep_by_col.values()) - - all_tasks = sorted({t for m in by_col.values() for t in m}, key=_task_sort_key) - cells: dict[str, dict[str, str]] = {} - raw: dict[str, dict[str, list[TaskResult]]] = {} - for tid in all_tasks: - cells[tid] = {} - raw[tid] = {} - for col in columns: - task_rows = by_col[col].get(tid, []) - raw[tid][col] = task_rows - if multi_rep: - cells[tid][col] = format_multi_rep_surface_cell(task_rows) - else: - cells[tid][col] = format_surface_cell(task_rows[-1] if task_rows else None) - - # Aggregate footer per column. - footer: dict[str, dict[str, Any]] = {} - for col in columns: - succ = run = calls = mispicks = 0 - mispick_comparable = True - infra = 0 - unstable_tasks = 0 - for _tid, task_rows in by_col[col].items(): - completed: list[TaskResult] = [] - for r in task_rows: - if is_infra_error_row(r): - infra += 1 - continue - if r.error: - continue - if r.skipped: - continue - completed.append(r) - run += 1 - if r.success: - succ += 1 - calls += r.num_calls - if r.classification == "external": - mispick_comparable = False - else: - alt, oos = r.alternate_calls, r.out_of_set_calls - if alt is None and oos is None: - mispick_comparable = False - else: - mispicks += int(alt or 0) + int(oos or 0) - task_k = sum(1 for r in completed if r.success) - if len(completed) > 1 and 0 < task_k < len(completed): - unstable_tasks += 1 - footer[col] = { - "success": succ, - "n": run, - "calls": calls, - "mispicks": mispicks if mispick_comparable else None, - "infra_errors": infra, - "multi_rep": multi_rep_by_col[col], - "unstable_tasks": unstable_tasks, - } - return { - "columns": columns, - "task_ids": all_tasks, - "cells": cells, - "raw": raw, - "footer": footer, - "multi_rep": multi_rep, - "multi_rep_by_col": multi_rep_by_col, - } - - -def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) -> str: - """Render multi-surface table as plain text or GitHub markdown.""" - cols: list[str] = table["columns"] - task_ids: list[str] = table["task_ids"] - cells: dict[str, dict[str, str]] = table["cells"] - footer: dict[str, dict[str, Any]] = table["footer"] - multi_rep = bool(table.get("multi_rep")) - lines: list[str] = [] - - def _prompt_snip(tid: str) -> str: - p = (TASKS_BY_ID.get(tid, {}).get("prompt") or "").replace("{project}", "P") - return (p[:32] + "…") if len(p) > 32 else p - - if markdown: - header = "| task | what | " + " | ".join(cols) + " |" - sep = "| --- | --- | " + " | ".join("---" for _ in cols) + " |" - lines.append(header) - lines.append(sep) - for tid in task_ids: - row_cells = " | ".join(cells[tid].get(c, "—") for c in cols) - lines.append(f"| {tid} | {_prompt_snip(tid)} | {row_cells} |") - # Footer - foot_parts = [] - for c in cols: - f = footer[c] - rate = f"{f['success']}/{f['n']}" if f["n"] else "0/0" - mp = f", {f['mispicks']}mp" if f["mispicks"] is not None else "" - foot_parts.append(f"{rate} ({f['calls']}c{mp}, i={f['infra_errors']})") - lines.append("| **agg** | | " + " | ".join(foot_parts) + " |") - if multi_rep: - noise_parts = [ - noise_floor_statement(int(footer[c].get("unstable_tasks") or 0)) - if footer[c].get("multi_rep") - else "single repetition" - for c in cols - ] - lines.append("| **noise floor** | | " + " | ".join(noise_parts) + " |") - return "\n".join(lines) + "\n" - - col_w = max(14, max((len(c) for c in cols), default=14)) - if multi_rep: - col_w = max(col_w, max((len(value) for task in cells.values() for value in task.values()), default=14)) - head = f"{'task':5} {'what':34} " + " ".join(f"{c:{col_w}}" for c in cols) - lines.append(head) - lines.append("-" * len(head)) - for tid in task_ids: - line = f"{tid:5} {_prompt_snip(tid):34} " - for c in cols: - line += f"{cells[tid].get(c, '—'):{col_w}} " - lines.append(line.rstrip()) - lines.append("-" * len(head)) - for c in cols: - f = footer[c] - rate = f"{f['success']}/{f['n']}" if f["n"] else "0/0" - pct = f" ({100 * f['success'] / f['n']:.0f}%)" if f["n"] else "" - mp = f" mispicks {f['mispicks']}" if f["mispicks"] is not None else " mispicks n/a" - lines.append(f"{c:12} success {rate}{pct} total calls {f['calls']}{mp} infra {f['infra_errors']}") - if multi_rep: - for c in cols: - if footer[c].get("multi_rep"): - lines.append(f"{c:12} {noise_floor_statement(int(footer[c].get('unstable_tasks') or 0))}") - return "\n".join(lines) + "\n" - - -def _surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: - """Pick a column label from the file's dominant surface field, else stem.""" - counts: dict[str, int] = defaultdict(int) - for r in rows: - s = r.surface - if s: - counts[str(s)] += 1 - if counts: - return max(counts, key=counts.get) # type: ignore[arg-type] - return path.stem - - -def warn_if_table_mixes_batteries(file_rows: list[tuple[str, list[ResultRow]]]) -> bool: - """Warn when table columns contain rows from different task batteries.""" - by_label: dict[str, set[str]] = {} - all_fingerprints: set[str] = set() - for label, rows in file_rows: - fingerprints = {_task_result(row).battery or "" for row in rows if not is_meta_row(row)} - if fingerprints: - by_label[label] = fingerprints - all_fingerprints.update(fingerprints) - if len(all_fingerprints) <= 1: - return False - detail = "; ".join(f"{label}={','.join(sorted(values))}" for label, values in by_label.items()) - print( - "warning: table spans battery fingerprints; these rows were graded on " - f"different task prompts/questions and are not directly comparable ({detail})", - file=sys.stderr, - ) - return True - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description="Summarize eval JSONL results") - p.add_argument( - "files", - nargs="*", - help="JSONL file(s): one for summary, two for A/B, N with --table", - ) - p.add_argument( - "--table", - action="store_true", - help="Multi-surface per-task table (one column per file, labeled by surface)", - ) - p.add_argument( - "--markdown", - action="store_true", - help="With --table, emit a GitHub-flavored markdown table", - ) - p.add_argument( - "--no-dedupe", - action="store_true", - help="Keep all rows (forensics); default is latest-wins per (task_id,rep,surface)", - ) - args = p.parse_args(argv) - dedupe: DedupeMode = "none" if args.no_dedupe else "latest" - - if not args.files: - p.print_help() - return 2 - - paths = [Path(f) for f in args.files] - for path in paths: - if not path.exists(): - print(f"error: file not found: {path}", file=sys.stderr) - return 2 - - if args.table: - if len(paths) < 1: - print("error: --table requires at least one JSONL", file=sys.stderr) - return 2 - labeled: list[tuple[str, list[TaskResult]]] = [] - used_labels: set[str] = set() - for path in paths: - rows = load_rows(path, dedupe=dedupe) - label = _surface_label_for_file(path, rows) - # Disambiguate duplicate surface labels (e.g. two external files). - base = label - n = 2 - while label in used_labels: - label = f"{base}-{n}" - n += 1 - used_labels.add(label) - labeled.append((label, rows)) - warn_if_table_mixes_batteries(labeled) - table = build_multi_surface_table(labeled) - sys.stdout.write(render_multi_surface_table(table, markdown=args.markdown)) - return 0 - - if len(paths) == 1: - path = paths[0] - rows = load_rows(path, dedupe=dedupe) - summary = summarize(rows) - task_keys = [k for k in summary if k != "_meta"] - if not task_keys: - infra_n = (summary.get("_meta") or {}).get("infra_errors", 0) - if infra_n: - print(f"infra errors: {infra_n}") - print(f"(no non-skipped / non-error rows in {path})") - return 0 - print_table(summary, f"Summary: {path}") - return 0 - - if len(paths) == 2: - rows_a = load_rows(paths[0], dedupe=dedupe) - rows_b = load_rows(paths[1], dedupe=dedupe) - cmp = ab_compare(rows_a, rows_b) - print_ab_report(cmp, paths[0], paths[1]) - return 0 - - print( - "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", - file=sys.stderr, - ) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evals/report/__init__.py b/evals/report/__init__.py new file mode 100644 index 00000000..34785668 --- /dev/null +++ b/evals/report/__init__.py @@ -0,0 +1,63 @@ +"""Evaluation result reports.""" + +from .command import main +from .compare import ab_compare, print_ab_report +from .load import ( + DedupeMode, + ResultRow, + dedupe_rows_latest, + is_infra_error_row, + is_meta_row, + load_rows, + read_result, +) +from .statistics import iqr, median, percentile, sign_test_pvalue, wilson_interval +from .summary import ResultTokensMode, noise_floor_statement, result_tokens_mode, summarize +from .table import ( + build_multi_surface_table, + format_multi_rep_surface_cell, + format_number, + format_result_tokens, + format_surface_cell, + print_table, + prompt_excerpt, + render_multi_surface_table, + result_tokens_marker, + surface_label_for_file, + task_sort_key, + warn_if_table_mixes_batteries, +) + +__all__ = [ + "DedupeMode", + "ResultRow", + "ResultTokensMode", + "ab_compare", + "build_multi_surface_table", + "dedupe_rows_latest", + "format_multi_rep_surface_cell", + "format_number", + "format_result_tokens", + "format_surface_cell", + "iqr", + "is_infra_error_row", + "is_meta_row", + "load_rows", + "main", + "median", + "noise_floor_statement", + "percentile", + "print_ab_report", + "print_table", + "prompt_excerpt", + "read_result", + "render_multi_surface_table", + "result_tokens_marker", + "result_tokens_mode", + "sign_test_pvalue", + "summarize", + "surface_label_for_file", + "task_sort_key", + "warn_if_table_mixes_batteries", + "wilson_interval", +] diff --git a/evals/report/__main__.py b/evals/report/__main__.py new file mode 100644 index 00000000..22e1c1f8 --- /dev/null +++ b/evals/report/__main__.py @@ -0,0 +1,13 @@ +"""Command-line entry for evaluation reports. + +Usage: + python -m evals.report evals/results/A.jsonl + python -m evals.report A.jsonl B.jsonl # A/B delta (sign test + Wilson) + python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface + python -m evals.report --table --markdown f1.jsonl f2.jsonl +""" + +from .command import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/report/command.py b/evals/report/command.py new file mode 100644 index 00000000..601189a7 --- /dev/null +++ b/evals/report/command.py @@ -0,0 +1,105 @@ +"""Command-line behavior for evaluation reports.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from evals.results import TaskResult + +from .compare import ab_compare, print_ab_report +from .load import DedupeMode, load_rows +from .summary import summarize +from .table import ( + build_multi_surface_table, + print_table, + render_multi_surface_table, + surface_label_for_file, + warn_if_table_mixes_batteries, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Summarize eval JSONL results") + parser.add_argument( + "files", + nargs="*", + help="JSONL file(s): one for summary, two for A/B, N with --table", + ) + parser.add_argument( + "--table", + action="store_true", + help="Multi-surface per-task table (one column per file, labeled by surface)", + ) + parser.add_argument( + "--markdown", + action="store_true", + help="With --table, emit a GitHub-flavored markdown table", + ) + parser.add_argument( + "--no-dedupe", + action="store_true", + help="Keep all rows (forensics); default is latest-wins per (task_id,rep,surface)", + ) + arguments = parser.parse_args(argv) + dedupe: DedupeMode = "none" if arguments.no_dedupe else "latest" + + if not arguments.files: + parser.print_help() + return 2 + + paths = [Path(file_name) for file_name in arguments.files] + for path in paths: + if not path.exists(): + print(f"error: file not found: {path}", file=sys.stderr) + return 2 + + if arguments.table: + if len(paths) < 1: + print("error: --table requires at least one JSONL", file=sys.stderr) + return 2 + labeled: list[tuple[str, list[TaskResult]]] = [] + used_labels: set[str] = set() + for path in paths: + rows = load_rows(path, dedupe=dedupe) + label = surface_label_for_file(path, rows) + # Disambiguate duplicate surface labels (e.g. two external files). + label_root = label + number = 2 + while label in used_labels: + label = f"{label_root}-{number}" + number += 1 + used_labels.add(label) + labeled.append((label, rows)) + warn_if_table_mixes_batteries(labeled) + table = build_multi_surface_table(labeled) + sys.stdout.write(render_multi_surface_table(table, markdown=arguments.markdown)) + return 0 + + if len(paths) == 1: + path = paths[0] + rows = load_rows(path, dedupe=dedupe) + summary = summarize(rows) + task_keys = [key for key in summary if key != "_meta"] + if not task_keys: + infrastructure_errors = (summary.get("_meta") or {}).get("infra_errors", 0) + if infrastructure_errors: + print(f"infra errors: {infrastructure_errors}") + print(f"(no non-skipped / non-error rows in {path})") + return 0 + print_table(summary, f"Summary: {path}") + return 0 + + if len(paths) == 2: + rows_a = load_rows(paths[0], dedupe=dedupe) + rows_b = load_rows(paths[1], dedupe=dedupe) + comparison = ab_compare(rows_a, rows_b) + print_ab_report(comparison, paths[0], paths[1]) + return 0 + + print( + "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", + file=sys.stderr, + ) + return 2 diff --git a/evals/report/compare.py b/evals/report/compare.py new file mode 100644 index 00000000..3c360424 --- /dev/null +++ b/evals/report/compare.py @@ -0,0 +1,122 @@ +"""A/B comparison for evaluation result sets.""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Any + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import median, sign_test_pvalue +from .summary import noise_floor_statement, summarize +from .table import format_number + + +def ab_compare( + rows_a: list[ResultRow], + rows_b: list[ResultRow], +) -> dict[str, Any]: + """Compare two result sets: paired call-count deltas + success rates. + + Paired call deltas only include tasks with at least one successful repetition + in both A and B. Calls are the median across successful repetitions; this is + identical to the historical behavior for single-rep files. + """ + summary_a = summarize(rows_a) + summary_b = summarize(rows_b) + + def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: + output: dict[str, list[float]] = defaultdict(list) + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + continue + if not row.success: + continue + output[row.task_id].append(float(row.num_calls)) + return dict(output) + + calls_a = successful_calls_by_task(rows_a) + calls_b = successful_calls_by_task(rows_b) + shared = sorted(set(calls_a) & set(calls_b)) + deltas: list[float] = [] + per_task: list[dict[str, Any]] = [] + for task_id in shared: + count_a = float(median(calls_a[task_id]) or 0.0) + count_b = float(median(calls_b[task_id]) or 0.0) + delta = count_b - count_a # B − A (negative = B fewer calls = better if lower is better) + deltas.append(delta) + per_task.append( + { + "task_id": task_id, + "calls_a": count_a, + "calls_b": count_b, + "delta": delta, + } + ) + + meta_a = summary_a.get("_meta") or {} + meta_b = summary_b.get("_meta") or {} + return { + "summary_a": summary_a, + "summary_b": summary_b, + "paired_tasks": per_task, + "median_delta": median(deltas), + "sign_test_p": sign_test_pvalue(deltas), + "n_paired": len(deltas), + "multi_rep": bool(meta_a.get("multi_rep") or meta_b.get("multi_rep")), + "unstable_a": int(meta_a.get("unstable_tasks") or 0), + "unstable_b": int(meta_b.get("unstable_tasks") or 0), + "success_a": { + "k": int(meta_a.get("aggregate_k") or 0), + "n": int(meta_a.get("aggregate_n") or 0), + "wilson": ( + float(meta_a.get("aggregate_wilson_lo") or 0.0), + float(meta_a.get("aggregate_wilson_hi") or 0.0), + ), + }, + "success_b": { + "k": int(meta_b.get("aggregate_k") or 0), + "n": int(meta_b.get("aggregate_n") or 0), + "wilson": ( + float(meta_b.get("aggregate_wilson_lo") or 0.0), + float(meta_b.get("aggregate_wilson_hi") or 0.0), + ), + }, + } + + +def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> None: + print(f"A/B compare: A={path_a} B={path_b}") + success_a, success_b = comparison["success_a"], comparison["success_b"] + rate_a = (success_a["k"] / success_a["n"]) if success_a["n"] else 0.0 + rate_b = (success_b["k"] / success_b["n"]) if success_b["n"] else 0.0 + print( + f" success A: {success_a['k']}/{success_a['n']} ({rate_a:.1%}) " + f"Wilson95 [{success_a['wilson'][0]:.2f},{success_a['wilson'][1]:.2f}]" + ) + print( + f" success B: {success_b['k']}/{success_b['n']} ({rate_b:.1%}) " + f"Wilson95 [{success_b['wilson'][0]:.2f},{success_b['wilson'][1]:.2f}]" + ) + print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") + print(f" paired successful tasks: {comparison['n_paired']}") + print(f" median call delta (B−A): {format_number(comparison['median_delta'])}") + probability = comparison["sign_test_p"] + print(f" sign-test p-value (two-sided): {probability if probability is not None else 'n/a'}") + multiple_repetitions = bool(comparison.get("multi_rep")) + if multiple_repetitions: + print(f" A {noise_floor_statement(int(comparison.get('unstable_a') or 0))}") + print(f" B {noise_floor_statement(int(comparison.get('unstable_b') or 0))}") + if comparison["paired_tasks"]: + print() + print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") + print("-" * 34) + for row in comparison["paired_tasks"]: + if multiple_repetitions: + print( + f"{row['task_id']:<6} {format_number(row['calls_a']):>8} " + f"{format_number(row['calls_b']):>8} {row['delta']:>+8.1f}" + ) + else: + print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") diff --git a/evals/report/load.py b/evals/report/load.py new file mode 100644 index 00000000..0a7a4425 --- /dev/null +++ b/evals/report/load.py @@ -0,0 +1,90 @@ +"""JSONL row loading and classification for evaluation reports.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any, Literal + +from evals.results import TaskResult + +DedupeMode = Literal["latest", "none"] +ResultRow = TaskResult | dict[str, Any] + + +def read_result(row: ResultRow) -> TaskResult: + """Return one row as the declared persisted result type.""" + return row if isinstance(row, TaskResult) else TaskResult.from_row(row) + + +def is_meta_row(row: ResultRow) -> bool: + """True for run-header meta lines (or any row without a task_id).""" + result = read_result(row) + return result.row_type == "meta" or not result.task_id + + +def is_infra_error_row(row: ResultRow) -> bool: + """True when a row failed for infrastructure reasons, not task verification. + + Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, + ``infra_api``, ``infra_sdk``, …) is excluded from success-rate denominators. + """ + error_class = read_result(row).error_class + return isinstance(error_class, str) and error_class.startswith("infra_") + + +def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: + """Keep only the last row per (task_id, rep, surface); preserve key insertion order.""" + latest: dict[tuple[str, int, str], TaskResult] = {} + order: list[tuple[str, int, str]] = [] + for raw_row in rows: + row = read_result(raw_row) + key = (row.task_id, row.rep, row.surface) + if key not in latest: + order.append(key) + latest[key] = row + return [latest[key] for key in order] + + +def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: + """Load JSONL data rows (skip meta / missing task_id). + + Default ``dedupe="latest"`` keeps the last row per (task_id, rep, surface) + so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. + """ + rows: list[TaskResult] = [] + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: {path}:{line_number}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict): + continue + result = TaskResult.from_row(row) + if is_meta_row(result): + continue + rows.append(result) + if dedupe == "latest": + return dedupe_rows_latest(rows) + # Forensics: warn on duplicates but keep all. + seen_keys: set[tuple[str, int, str]] = set() + for row in rows: + key = (row.task_id, row.rep, row.surface) + if key in seen_keys: + print( + f"warning: {path}: duplicate (task_id, rep, surface)={key} " + f"(--no-dedupe keeps all rows; bare --out reuse double-counts)", + file=sys.stderr, + ) + else: + seen_keys.add(key) + return rows diff --git a/evals/report/statistics.py b/evals/report/statistics.py new file mode 100644 index 00000000..33ee0546 --- /dev/null +++ b/evals/report/statistics.py @@ -0,0 +1,65 @@ +"""Statistical calculations for evaluation reports.""" + +from __future__ import annotations + +import math + + +def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """95% Wilson score interval for a binomial proportion.""" + if n <= 0: + return (0.0, 0.0) + p = k / n + z2 = z * z + denom = 1.0 + z2 / n + centre = p + z2 / (2.0 * n) + margin = z * math.sqrt((p * (1.0 - p) + z2 / (4.0 * n)) / n) + lo = max(0.0, (centre - margin) / denom) + hi = min(1.0, (centre + margin) / denom) + return (lo, hi) + + +def sign_test_pvalue(deltas: list[float]) -> float | None: + """Two-sided exact binomial sign test on non-zero paired deltas. + + H0: P(delta > 0) = 1/2. Zero deltas are dropped. Returns None when no + non-zero pairs remain. Uses ``math.comb`` only (no scipy). + """ + nonzero = [d for d in deltas if d != 0] + n = len(nonzero) + if n == 0: + return None + k = sum(1 for d in nonzero if d > 0) + total = 2**n + # Two-sided: 2 * min(left cdf, right survival), capped at 1. + left = sum(math.comb(n, i) for i in range(0, k + 1)) / total + right = sum(math.comb(n, i) for i in range(k, n + 1)) / total + return min(1.0, 2.0 * min(left, right)) + + +def median(values: list[float]) -> float | None: + if not values: + return None + sorted_values = sorted(values) + middle = len(sorted_values) // 2 + if len(sorted_values) % 2: + return float(sorted_values[middle]) + return (sorted_values[middle - 1] + sorted_values[middle]) / 2.0 + + +def percentile(values: list[float], proportion: float) -> float | None: + if not values: + return None + sorted_values = sorted(values) + if len(sorted_values) == 1: + return float(sorted_values[0]) + position = (len(sorted_values) - 1) * proportion + floor = math.floor(position) + ceiling = math.ceil(position) + if floor == ceiling: + return float(sorted_values[int(position)]) + return float(sorted_values[floor] + (sorted_values[ceiling] - sorted_values[floor]) * (position - floor)) + + +def iqr(values: list[float]) -> tuple[float | None, float | None, float | None]: + return (percentile(values, 0.25), median(values), percentile(values, 0.75)) diff --git a/evals/report/summary.py b/evals/report/summary.py new file mode 100644 index 00000000..2ce90657 --- /dev/null +++ b/evals/report/summary.py @@ -0,0 +1,168 @@ +"""Aggregate evaluation results and measure observed task instability.""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Literal + +from evals.results import TaskResult +from evals.tasks import TASKS_BY_ID + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import iqr, median, percentile, wilson_interval + +ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] + + +def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: + """Classify token counts without treating unmarked legacy data as measured.""" + labels: set[str] = set() + for raw_row in rows: + row = read_result(raw_row) + for call in row.calls: + if call.result_tokens is None: + continue + estimated = ( + call.result_tokens_estimated + if call.result_tokens_estimated is not None + else row.result_tokens_estimated + ) + if estimated is True: + labels.add("estimated") + elif estimated is False: + labels.add("measured") + else: + labels.add("unlabeled") + if not labels: + return "unavailable" + if labels == {"estimated"}: + return "estimated" + if labels == {"measured"}: + return "measured" + if "unlabeled" in labels: + return "unlabeled" + return "mixed" + + +def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: + """Aggregate per-task metrics. + + Rows with ``error_class`` starting ``infra_`` are excluded from success-rate + denominators and counted separately as ``infra_errors`` (total on the returned + dict under the special key ``_meta``). Other non-null ``error`` rows remain + harness errors (excluded from success, counted in ``harness_err``). + """ + by_task: dict[str, list[TaskResult]] = defaultdict(list) + harness_errors_by_task: dict[str, int] = defaultdict(int) + infrastructure_errors_by_task: dict[str, int] = defaultdict(int) + repetitions_by_task: dict[str, set[int]] = defaultdict(set) + infrastructure_errors = 0 + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row): + continue + task_id = row.task_id + repetitions_by_task[task_id].add(row.rep) + if is_infra_error_row(row): + infrastructure_errors += 1 + infrastructure_errors_by_task[task_id] += 1 + continue # infra seed/cli — excluded from success aggregates + if row.error: + harness_errors_by_task[task_id] += 1 + continue # harness/API errors excluded from success/medians (F4) + if row.skipped: + continue # skipped rows are excluded from success denominators + by_task[task_id].append(row) + + # Include tasks that only had harness/infra errors so columns stay visible. + all_task_ids = sorted(set(by_task) | set(harness_errors_by_task) | set(infrastructure_errors_by_task)) + + output: dict[str, dict[str, Any]] = {} + total_passes = 0 + total_repetitions = 0 + for task_id in all_task_ids: + task_results = by_task.get(task_id, []) + repetition_count = len(task_results) + pass_count = sum(1 for row in task_results if row.success) + unstable = repetition_count > 1 and 0 < pass_count < repetition_count + total_passes += pass_count + total_repetitions += repetition_count + lower, upper = wilson_interval(pass_count, repetition_count) if repetition_count else (0.0, 0.0) + calls = [float(row.num_calls) for row in task_results] + first_quartile, median_calls, third_quartile = iqr(calls) + minimum_calls = min(calls) if calls else None + maximum_calls = max(calls) if calls else None + optimal = TASKS_BY_ID.get(task_id, {}).get("optimal_calls") + total_calls = 0 + mispicks = 0 + errored_calls = 0 + result_tokens: list[float] = [] + for row in task_results: + for call in row.calls: + total_calls += 1 + if call.classification in ("alternate", "out_of_set"): + mispicks += 1 + if call.is_error: + errored_calls += 1 + if call.result_tokens is not None: + result_tokens.append(float(call.result_tokens)) + capped = sum(1 for row in task_results if row.hit_max_iterations or row.stop_reason == "max_tokens") + cumulative_inputs = [float(row.cum_input_tokens or 0) for row in task_results] + output[task_id] = { + "n": repetition_count, + "k": pass_count, + "success": f"{pass_count}/{repetition_count}" if repetition_count else "0/0", + "unstable": unstable, + "wilson_lo": lower, + "wilson_hi": upper, + "med_calls": median_calls, + "calls_min": minimum_calls, + "calls_max": maximum_calls, + "calls_q1": first_quartile, + "calls_q3": third_quartile, + "optimal_calls": optimal, + "mispick_rate": (mispicks / total_calls) if total_calls else 0.0, + "errored_calls": errored_calls, + "capped": capped, + "harness_err": harness_errors_by_task.get(task_id, 0), + "infra_err": infrastructure_errors_by_task.get(task_id, 0), + "med_result_tokens": median(result_tokens), + "p95_result_tokens": percentile(result_tokens, 0.95), + "result_tokens_mode": result_tokens_mode(task_results), + "med_cum_input": median(cumulative_inputs), + } + aggregate_lower, aggregate_upper = ( + wilson_interval(total_passes, total_repetitions) if total_repetitions else (0.0, 0.0) + ) + unstable_task_ids = sorted(task_id for task_id, values in output.items() if values.get("unstable")) + output["_meta"] = { + "infra_errors": infrastructure_errors, + "aggregate_k": total_passes, + "aggregate_n": total_repetitions, + "aggregate_wilson_lo": aggregate_lower, + "aggregate_wilson_hi": aggregate_upper, + "multi_rep": any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), + "unstable_task_ids": unstable_task_ids, + "unstable_tasks": len(unstable_task_ids), + "result_tokens_mode": result_tokens_mode([row for task_results in by_task.values() for row in task_results]), + } + return output + + +def noise_floor_statement(unstable_tasks: int) -> str: + """Describe observed pass/fail variance in task-count comparison units.""" + count = max(0, int(unstable_tasks)) + if count == 0: + return ( + "measured noise floor: 0 tasks flipped at least once; no non-zero " + "run-to-run variance was observed (minimum meaningful difference " + "from observed flips: 1 task)" + ) + noun = "task" if count == 1 else "tasks" + threshold = count + 1 + threshold_noun = "task" if threshold == 1 else "tasks" + return ( + f"measured noise floor: {count} {noun} flipped at least once; surface " + f"differences of {count} {noun} or fewer are within observed run-to-run " + f"variance (minimum meaningful difference: {threshold} {threshold_noun})" + ) diff --git a/evals/report/table.py b/evals/report/table.py new file mode 100644 index 00000000..a248e6c2 --- /dev/null +++ b/evals/report/table.py @@ -0,0 +1,379 @@ +"""Plain-text and Markdown tables for evaluation reports.""" + +from __future__ import annotations + +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +from evals.results import TaskResult +from evals.tasks import TASKS_BY_ID + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import wilson_interval +from .summary import noise_floor_statement + + +def format_number(value: float | None, digits: int = 1) -> str: + if value is None: + return "-" + return f"{value:.{digits}f}" + + +def result_tokens_marker(mode: str) -> str: + return {"estimated": "~", "mixed": "*", "unlabeled": "?"}.get(mode, "") + + +def format_result_tokens(value: float | None, mode: str) -> str: + formatted = format_number(value, 0) + if formatted == "-": + return formatted + return f"{result_tokens_marker(mode)}{formatted}" + + +def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: + meta = summary.get("_meta") or {} + print(title) + token_mode = str(meta.get("result_tokens_mode") or "unavailable") + if token_mode == "estimated": + print("result-token columns marked ~: entirely estimated from result characters") + elif token_mode == "mixed": + print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") + elif token_mode == "unlabeled": + print("result-token columns marked ?: include legacy values with unknown measurement status") + if meta.get("infra_errors"): + print(f"infra errors: {meta['infra_errors']}") + aggregate_count = int(meta.get("aggregate_n") or 0) + if aggregate_count: + aggregate_passes = int(meta.get("aggregate_k") or 0) + lower = float(meta.get("aggregate_wilson_lo") or 0.0) + upper = float(meta.get("aggregate_wilson_hi") or 0.0) + rate = aggregate_passes / aggregate_count if aggregate_count else 0.0 + print( + f"aggregate success: {aggregate_passes}/{aggregate_count} ({rate:.1%}) Wilson95 [{lower:.2f},{upper:.2f}]" + ) + multiple_repetitions = bool(meta.get("multi_rep")) + if multiple_repetitions: + print(noise_floor_statement(int(meta.get("unstable_tasks") or 0))) + # Multi-rep files keep the repetition-aware layout even when errors leave + # only one completed result in every task's success-rate denominator. + show_variation = multiple_repetitions or any( + values.get("n", 0) > 1 for task_id, values in summary.items() if task_id != "_meta" + ) + token_marker = result_tokens_marker(token_mode) + median_result_tokens_header = f"med_rtok{token_marker}" + percentile_result_tokens_header = f"p95_rtok{token_marker}" + if show_variation: + unstable_header = f"{'unstable':>8} " if multiple_repetitions else "" + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{unstable_header}" + f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " + f"{'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} " + f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " + f"{'med_cum_in':>10}" + ) + else: + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{'med_calls':>9} {'opt':>4} {'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} " + f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " + f"{'med_cum_in':>10}" + ) + print(header) + print("-" * len(header)) + for task_id, values in summary.items(): + if task_id == "_meta": + continue + wilson = f"[{values['wilson_lo']:.2f},{values['wilson_hi']:.2f}]" + quartiles = f"{format_number(values['calls_q1'])}-{format_number(values['calls_q3'])}" + optimal = values["optimal_calls"] if values["optimal_calls"] is not None else "-" + task_token_mode = str(values.get("result_tokens_mode") or "unavailable") + if show_variation: + unstable = ("YES" if values.get("unstable") else "no") if multiple_repetitions else "" + unstable_cell = f"{unstable:>8} " if multiple_repetitions else "" + print( + f"{task_id:<6} {values['n']:>3} {values['success']:>8} {wilson:>16} " + f"{unstable_cell}" + f"{format_number(values.get('calls_min')):>9} " + f"{format_number(values['med_calls']):>9} " + f"{format_number(values.get('calls_max')):>9} " + f"{optimal!s:>4} {quartiles:>11} {values['mispick_rate']:>7.1%} " + f"{values['errored_calls']:>4} {values['capped']:>6} " + f"{values['harness_err']:>5} {values.get('infra_err', 0):>5} " + f"{format_result_tokens(values['med_result_tokens'], task_token_mode):>9} " + f"{format_result_tokens(values['p95_result_tokens'], task_token_mode):>9} " + f"{format_number(values['med_cum_input'], 0):>10}" + ) + else: + print( + f"{task_id:<6} {values['n']:>3} {values['success']:>8} {wilson:>16} " + f"{format_number(values['med_calls']):>9} {optimal!s:>4} " + f"{quartiles:>11} {values['mispick_rate']:>7.1%} " + f"{values['errored_calls']:>4} {values['capped']:>6} " + f"{values['harness_err']:>5} {values.get('infra_err', 0):>5} " + f"{format_result_tokens(values['med_result_tokens'], task_token_mode):>9} " + f"{format_result_tokens(values['p95_result_tokens'], task_token_mode):>9} " + f"{format_number(values['med_cum_input'], 0):>10}" + ) + + +def task_sort_key(task_id: str) -> tuple[str, int]: + digits = "".join(character for character in task_id if character.isdigit()) + return (task_id[0] if task_id else "", int(digits) if digits else 0) + + +def format_surface_cell(row: ResultRow | None) -> str: + """Cell for multi-surface table: '✅ Nc/Mmp', 'skip', 'ERR', or '—'.""" + if row is None: + return "—" + result = read_result(row) + if result.skipped: + return "skip" + if result.error or is_infra_error_row(result): + return "ERR" + passed = "✅" if result.success else "❌" + call_count = str(result.num_calls) + if result.classification == "external": + return f"{passed} {call_count}c" + alternate = result.alternate_calls + outside_set = result.out_of_set_calls + # None counters (external nulling) → omit mispick suffix. + if alternate is None and outside_set is None: + return f"{passed} {call_count}c" + mispicks = int(alternate or 0) + int(outside_set or 0) + if mispicks: + return f"{passed} {call_count}c/{mispicks}mp" + return f"{passed} {call_count}c" + + +def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: + """Aggregate distinct repetitions into one task/surface cell.""" + results = [read_result(row) for row in rows] + completed = [row for row in results if not is_infra_error_row(row) and not row.error and not row.skipped] + if completed: + repetition_count = len(completed) + pass_count = sum(1 for row in completed if row.success) + lower, upper = wilson_interval(pass_count, repetition_count) + if 0 < pass_count < repetition_count: + marker = "⚠ UNSTABLE" + elif pass_count == repetition_count: + marker = "✅" + else: + marker = "❌" + calls = [row.num_calls for row in completed] + call_span = f"{min(calls)}c" if min(calls) == max(calls) else f"{min(calls)}-{max(calls)}c" + return f"{marker} {pass_count}/{repetition_count} [{lower:.2f},{upper:.2f}] {call_span}" + if any(row.error or is_infra_error_row(row) for row in results): + return "ERR" + if any(row.skipped for row in results): + return "skip" + return "—" + + +def build_multi_surface_table( + file_rows: list[tuple[str, list[ResultRow]]], +) -> dict[str, Any]: + """Build a per-task × per-surface grid from labeled row sets. + + ``file_rows`` is a list of ``(column_label, rows)``. Column labels default + to each file's dominant ``surface`` field when the caller passes that label. + Rows are grouped by task and repetition. Single-rep columns retain the + historical one-cell rendering; multi-rep columns aggregate all repetitions. + """ + columns: list[str] = [] + rows_by_column: dict[str, dict[str, list[TaskResult]]] = {} + multiple_repetitions_by_column: dict[str, bool] = {} + for label, rows in file_rows: + columns.append(label) + column_rows: dict[str, list[TaskResult]] = defaultdict(list) + for raw_row in rows: + if is_meta_row(raw_row): + continue + row = read_result(raw_row) + column_rows[row.task_id].append(row) + rows_by_column[label] = dict(column_rows) + multiple_repetitions_by_column[label] = any( + len({row.rep for row in task_rows}) > 1 for task_rows in column_rows.values() + ) + + multiple_repetitions = any(multiple_repetitions_by_column.values()) + + all_tasks = sorted( + {task_id for column in rows_by_column.values() for task_id in column}, + key=task_sort_key, + ) + cells: dict[str, dict[str, str]] = {} + raw: dict[str, dict[str, list[TaskResult]]] = {} + for task_id in all_tasks: + cells[task_id] = {} + raw[task_id] = {} + for column in columns: + task_rows = rows_by_column[column].get(task_id, []) + raw[task_id][column] = task_rows + if multiple_repetitions: + cells[task_id][column] = format_multi_rep_surface_cell(task_rows) + else: + cells[task_id][column] = format_surface_cell(task_rows[-1] if task_rows else None) + + # Aggregate footer per column. + footer: dict[str, dict[str, Any]] = {} + for column in columns: + successes = repetitions = calls = mispicks = 0 + mispicks_comparable = True + infrastructure_errors = 0 + unstable_tasks = 0 + for task_rows in rows_by_column[column].values(): + completed: list[TaskResult] = [] + for row in task_rows: + if is_infra_error_row(row): + infrastructure_errors += 1 + continue + if row.error: + continue + if row.skipped: + continue + completed.append(row) + repetitions += 1 + if row.success: + successes += 1 + calls += row.num_calls + if row.classification == "external": + mispicks_comparable = False + else: + alternate = row.alternate_calls + outside_set = row.out_of_set_calls + if alternate is None and outside_set is None: + mispicks_comparable = False + else: + mispicks += int(alternate or 0) + int(outside_set or 0) + task_passes = sum(1 for row in completed if row.success) + if len(completed) > 1 and 0 < task_passes < len(completed): + unstable_tasks += 1 + footer[column] = { + "success": successes, + "n": repetitions, + "calls": calls, + "mispicks": mispicks if mispicks_comparable else None, + "infra_errors": infrastructure_errors, + "multi_rep": multiple_repetitions_by_column[column], + "unstable_tasks": unstable_tasks, + } + return { + "columns": columns, + "task_ids": all_tasks, + "cells": cells, + "raw": raw, + "footer": footer, + "multi_rep": multiple_repetitions, + "multi_rep_by_col": multiple_repetitions_by_column, + } + + +def prompt_excerpt(task_id: str) -> str: + prompt = (TASKS_BY_ID.get(task_id, {}).get("prompt") or "").replace("{project}", "P") + return (prompt[:32] + "…") if len(prompt) > 32 else prompt + + +def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) -> str: + """Render multi-surface table as plain text or GitHub markdown.""" + columns: list[str] = table["columns"] + task_ids: list[str] = table["task_ids"] + cells: dict[str, dict[str, str]] = table["cells"] + footer: dict[str, dict[str, Any]] = table["footer"] + multiple_repetitions = bool(table.get("multi_rep")) + lines: list[str] = [] + + if markdown: + header = "| task | what | " + " | ".join(columns) + " |" + separator = "| --- | --- | " + " | ".join("---" for _ in columns) + " |" + lines.append(header) + lines.append(separator) + for task_id in task_ids: + row_cells = " | ".join(cells[task_id].get(column, "—") for column in columns) + lines.append(f"| {task_id} | {prompt_excerpt(task_id)} | {row_cells} |") + # Footer + footer_parts = [] + for column in columns: + values = footer[column] + rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + mispicks = f", {values['mispicks']}mp" if values["mispicks"] is not None else "" + footer_parts.append(f"{rate} ({values['calls']}c{mispicks}, i={values['infra_errors']})") + lines.append("| **agg** | | " + " | ".join(footer_parts) + " |") + if multiple_repetitions: + noise_parts = [ + noise_floor_statement(int(footer[column].get("unstable_tasks") or 0)) + if footer[column].get("multi_rep") + else "single repetition" + for column in columns + ] + lines.append("| **noise floor** | | " + " | ".join(noise_parts) + " |") + return "\n".join(lines) + "\n" + + column_width = max(14, max((len(column) for column in columns), default=14)) + if multiple_repetitions: + column_width = max( + column_width, + max( + (len(value) for task in cells.values() for value in task.values()), + default=14, + ), + ) + heading = f"{'task':5} {'what':34} " + " ".join(f"{column:{column_width}}" for column in columns) + lines.append(heading) + lines.append("-" * len(heading)) + for task_id in task_ids: + line = f"{task_id:5} {prompt_excerpt(task_id):34} " + for column in columns: + line += f"{cells[task_id].get(column, '—'):{column_width}} " + lines.append(line.rstrip()) + lines.append("-" * len(heading)) + for column in columns: + values = footer[column] + rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + percentage = f" ({100 * values['success'] / values['n']:.0f}%)" if values["n"] else "" + mispicks = f" mispicks {values['mispicks']}" if values["mispicks"] is not None else " mispicks n/a" + lines.append( + f"{column:12} success {rate}{percentage} total calls {values['calls']}" + f"{mispicks} infra {values['infra_errors']}" + ) + if multiple_repetitions: + for column in columns: + if footer[column].get("multi_rep"): + lines.append(f"{column:12} {noise_floor_statement(int(footer[column].get('unstable_tasks') or 0))}") + return "\n".join(lines) + "\n" + + +def surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: + """Pick a column label from the file's dominant surface field, else stem.""" + counts: dict[str, int] = defaultdict(int) + for row in rows: + surface = row.surface + if surface: + counts[str(surface)] += 1 + if counts: + return max(counts, key=counts.get) # type: ignore[arg-type] + return path.stem + + +def warn_if_table_mixes_batteries(file_rows: list[tuple[str, list[ResultRow]]]) -> bool: + """Warn when table columns contain rows from different task batteries.""" + by_label: dict[str, set[str]] = {} + all_fingerprints: set[str] = set() + for label, rows in file_rows: + fingerprints = {read_result(row).battery or "" for row in rows if not is_meta_row(row)} + if fingerprints: + by_label[label] = fingerprints + all_fingerprints.update(fingerprints) + if len(all_fingerprints) <= 1: + return False + detail = "; ".join(f"{label}={','.join(sorted(values))}" for label, values in by_label.items()) + print( + "warning: table spans battery fingerprints; these rows were graded on " + f"different task prompts/questions and are not directly comparable ({detail})", + file=sys.stderr, + ) + return True diff --git a/evals/results.py b/evals/results.py index 1a6c60a0..cee442bb 100644 --- a/evals/results.py +++ b/evals/results.py @@ -50,6 +50,7 @@ class TaskResult: """ schema_version: int = RESULT_SCHEMA_VERSION + row_type: str | None = None run_id: str = "" ts: str = "" git_sha: str = "" @@ -231,6 +232,8 @@ def usage_row(item: Usage) -> dict[str, int]: "usage": usage_row(self.usage) if isinstance(self.usage, Usage) else self.usage, "usage_total": self.usage_total, } + if self.row_type is not None: + row["row_type"] = self.row_type if self.result_tokens_skipped_reason is not None: row["result_tokens_skipped_reason"] = self.result_tokens_skipped_reason return row @@ -300,6 +303,7 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: out_of_set_default = sum(1 for call in calls if call.classification == "out_of_set") return cls( schema_version=int(row.get("schema_version") or 0), + row_type=(str(row["row_type"]) if row.get("row_type") is not None else None), run_id=str(row.get("run_id") or ""), ts=str(row.get("ts") or ""), git_sha=str(row.get("git_sha") or ""), diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index 13b303a7..b41a0ac2 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -113,6 +113,7 @@ def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): def test_task_result_schema_round_trip_owns_usage_shape(): result = TaskResult( + row_type="result", task_id="R1", calls=[ CallRecord( @@ -129,8 +130,10 @@ def test_task_result_schema_round_trip_owns_usage_shape(): row = result.to_row() assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["row_type"] == "result" assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] loaded = TaskResult.from_row(row) + assert loaded.row_type == "result" assert loaded.calls[0].tool == "find_work_items" assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] From 5dbdbff15af3d61694d072fb325e703da56c7218 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 15:08:04 +0530 Subject: [PATCH 15/93] Group the run loop into a package and place helpers with their users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes that all reduce noise rather than move it around. runner.py was 753 lines with run_live about 300 of them: seeding, driving, verification, row assembly, teardown and resume bookkeeping interleaved in one function, which made it the hardest code here to follow. It is now a package — live, resume, meta, canary — and a single repetition runs through its own function with a thin loop around it. Splitting it into root modules would have traded one problem for another, so the package keeps evals/ root from growing and every existing import still resolves. token_counting moves into drivers, whose two modules are its only consumers. The rule we settled on is that a helper lives with its user and graduates to the root when a second package needs it; keeping it at the top level advertised a shared dependency that did not exist. The default output directory becomes evals/output, because evals/results.py and evals/results/ differed only in punctuation. An existing results directory is left on disk untouched — it may hold earlier runs — and stays ignored. Verified through the real loop, not fakes: the live canary still rejects a do-nothing agent on all 33 tasks, and a live two-task run passes with nothing left behind in the workspace. A decomposition that reordered teardown against verification would pass the offline suite and only fail on a battery. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + evals/DESIGN.md | 13 +- evals/README.md | 18 +- evals/cli.py | 2 +- evals/drivers/api/driver.py | 2 +- evals/drivers/base.py | 4 +- evals/{ => drivers}/token_counting.py | 2 +- evals/report/__main__.py | 2 +- evals/runner.py | 753 -------------------------- evals/runner/__init__.py | 32 ++ evals/runner/canary.py | 90 +++ evals/runner/live.py | 519 ++++++++++++++++++ evals/runner/meta.py | 75 +++ evals/runner/resume.py | 124 +++++ tests/test_evals_api_driver.py | 2 +- tests/test_evals_drivers.py | 2 +- tests/test_evals_hardening.py | 87 +-- tests/test_evals_proxy.py | 4 +- tests/test_evals_surface.py | 2 +- 19 files changed, 917 insertions(+), 818 deletions(-) rename evals/{ => drivers}/token_counting.py (96%) delete mode 100644 evals/runner.py create mode 100644 evals/runner/__init__.py create mode 100644 evals/runner/canary.py create mode 100644 evals/runner/live.py create mode 100644 evals/runner/meta.py create mode 100644 evals/runner/resume.py diff --git a/.gitignore b/.gitignore index c771e92d..fee4f34a 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ htmlcov/ .hypothesis/ # Eval harness output + local env bootstrap state +evals/output/ +# Keep earlier local runs ignored after the default directory rename. evals/results/ evals/.env-pids evals/.api_runserver.log diff --git a/evals/DESIGN.md b/evals/DESIGN.md index f5813a19..07d5afc9 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -194,7 +194,12 @@ rows preserve the common fields consumed by `evals.report` and existing JSONL re ```text evals/ cli.py argparse, command dispatch, and model-tier resolution - runner.py live lifecycle, row assembly, resume/meta handling, and canary + runner/ + __init__.py public execution API + live.py live lifecycle and row assembly + resume.py resume skip and mismatch checks + meta.py run metadata and repository provenance + canary.py empty-agent verifier canary run.py compatibility entry point for python -m evals.run tasks/ __init__.py ordered catalog assembly and public task API @@ -207,6 +212,7 @@ evals/ drivers/ __init__.py public exports and driver registry base.py AgentDriver, AgentRun, normalization, and common row mapping + token_counting.py tool-result token sizing claude.py Claude Code CLI driver codex.py Codex CLI driver antigravity.py Antigravity CLI driver @@ -219,9 +225,8 @@ evals/ anthropic.py Anthropic Messages translation openai.py OpenAI Chat Completions translation proxy.py stdlib-only JSON-RPC recording relay - seed.py Plane fixture creation and teardown - report.py summaries, A/B comparison, and multi-surface tables - token_counting.py shared estimate and optional local tokenizer counting + seed/ Plane fixture creation and teardown + report/ summaries, A/B comparison, and multi-surface tables ``` The stable import and command surfaces are intentional: `from evals.tasks import ...`, diff --git a/evals/README.md b/evals/README.md index 4f8ce1fd..2729f5fb 100644 --- a/evals/README.md +++ b/evals/README.md @@ -44,20 +44,20 @@ agent's final text. ```bash # Provider-neutral API loop (default provider: Anthropic) .venv/bin/python -m evals.run --driver api --provider anthropic --model standard \ - --surface full --out results/api.jsonl + --surface full --out evals/output/api.jsonl # Everything, one surface (free-form model IDs pass through to the CLI) .venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ - --surface full --out results/legacy.jsonl + --surface full --out evals/output/legacy.jsonl # A few tasks while iterating .venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ - --surface full --tasks W5,W8 --out results/spot.jsonl + --surface full --tasks W5,W8 --out evals/output/spot.jsonl # Someone else's server (a PR branch, another repo) — "external mode" .venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ --surface their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ - --server-env PLANE_MCP_TOOLS_VERSION=v2 --out results/their-pr.jsonl + --server-env PLANE_MCP_TOOLS_VERSION=v2 --out evals/output/their-pr.jsonl ``` Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed or @@ -133,9 +133,9 @@ habit; use it only when the more sensitive, larger sidecar is justified. ### Reading results ```bash -.venv/bin/python -m evals.report results/legacy.jsonl # one surface -.venv/bin/python -m evals.report --table results/*.jsonl # side by side -.venv/bin/python -m evals.report --table --markdown results/*.jsonl # for a PR +.venv/bin/python -m evals.report evals/output/legacy.jsonl # one surface +.venv/bin/python -m evals.report --table evals/output/*.jsonl # side by side +.venv/bin/python -m evals.report --table --markdown evals/output/*.jsonl # for a PR ``` Rows are deduped latest-wins per `(task_id, rep, surface)`, so a re-run of a single task @@ -164,8 +164,8 @@ Tasks that touch **workspace-scoped** fixtures (release tags, customer propertie if two runs share a workspace. Give each concurrent run its own workspace: ```bash -EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --surface full --out results/legacy.jsonl & -EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --surface their-pr --out results/their-pr.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --surface full --out evals/output/legacy.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --surface their-pr --out evals/output/their-pr.jsonl & wait ``` diff --git a/evals/cli.py b/evals/cli.py index fc6c2c96..c4211766 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -56,7 +56,7 @@ "opencode-cli": {}, } -DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "results" +DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "output" def resolve_model_for_driver(driver_name: str, model: str, *, provider: str | None = None) -> str: diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 10f17cc4..85a27b94 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -24,8 +24,8 @@ create_backend, ) from evals.drivers.base import AgentRun +from evals.drivers.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens from evals.results import Usage -from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens DEFAULT_MAX_TOKENS = 8192 diff --git a/evals/drivers/base.py b/evals/drivers/base.py index 1b604223..d7f7bd53 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -9,12 +9,12 @@ from pathlib import Path from typing import Any, Protocol -from evals.results import CallRecord, TaskResult, Usage -from evals.token_counting import ( +from evals.drivers.token_counting import ( TOKEN_ESTIMATE_METHOD, count_result_text_tokens, estimate_result_tokens, ) +from evals.results import CallRecord, TaskResult, Usage REPO_ROOT = Path(__file__).resolve().parent.parent.parent diff --git a/evals/token_counting.py b/evals/drivers/token_counting.py similarity index 96% rename from evals/token_counting.py rename to evals/drivers/token_counting.py index cc20dada..dd286619 100644 --- a/evals/token_counting.py +++ b/evals/drivers/token_counting.py @@ -1,4 +1,4 @@ -"""Shared tool-result token sizing for eval drivers and analysis.""" +"""Tool-result token sizing for evaluation drivers.""" from __future__ import annotations diff --git a/evals/report/__main__.py b/evals/report/__main__.py index 22e1c1f8..e529f94a 100644 --- a/evals/report/__main__.py +++ b/evals/report/__main__.py @@ -1,7 +1,7 @@ """Command-line entry for evaluation reports. Usage: - python -m evals.report evals/results/A.jsonl + python -m evals.report evals/output/A.jsonl python -m evals.report A.jsonl B.jsonl # A/B delta (sign test + Wilson) python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface python -m evals.report --table --markdown f1.jsonl f2.jsonl diff --git a/evals/runner.py b/evals/runner.py deleted file mode 100644 index 91a44172..00000000 --- a/evals/runner.py +++ /dev/null @@ -1,753 +0,0 @@ -"""Live execution, resume bookkeeping, row assembly, and verifier canary.""" - -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from evals.drivers import ( - KNOWN_DRIVERS, - agent_run_to_task_result, - get_driver, -) -from evals.drivers.api import MODEL_TIERS -from evals.results import RESULT_SCHEMA_VERSION, TaskResult -from evals.seed import make_plane_client, seed, teardown -from evals.tasks import ( - PromptBindError, - TaskSkipped, - battery_fingerprint, - format_task_prompt, - resolve_surface_tool_sets, - task_author, -) - -# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). -# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env -# (see plane_mcp.v2.choose_stdio_mcp). -KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) - -MAX_ITERATIONS = 15 -MAX_TOKENS = 8192 - - -def _git_sha() -> str: - try: - return ( - subprocess.check_output( - ["git", "rev-parse", "HEAD"], - stderr=subprocess.DEVNULL, - cwd=Path(__file__).resolve().parent.parent, - ) - .decode() - .strip() - ) - except Exception: - return "unknown" - - -def _system_preamble(workspace_slug: str, project_name: str) -> str: - """Keep under 100 words — part of measured context.""" - return ( - f"You are evaluating Plane project management tools. " - f"Workspace slug: {workspace_slug}. Project name: {project_name}. " - f"Complete the task using the available tools, then stop." - ) - - -def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: - if tool in optimal: - return "optimal" - if tool in alternate: - return "alternate" - return "out_of_set" - - -def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: - """Build MCP stdio env from scratch — never inherit os.environ (F6). - - ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the - v2 tool registry. ``surface=full`` leaves the var unset (legacy default). - Other surface names (external servers under benchmark) set nothing; their - selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. - """ - env: dict[str, str] = {} - if path := os.environ.get("PATH"): - env["PATH"] = path - if home := os.environ.get("HOME"): - env["HOME"] = home - env["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] - env["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] - env["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") - if surface == "v2": - env["PLANE_MCP_SURFACE"] = "v2" - elif surface == "v2-schema": - env["PLANE_MCP_SURFACE"] = "v2-schema" - if extra: - env.update(extra) - return env - - -def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: - """Return True if a prior row is a completed result that resume should skip. - - Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. - Rows with ``skipped`` set are treated as complete and are not retried (intentional: - surface/plan skips are stable outcomes, not infra failures). - Pure function — unit-tested without the live battery. - """ - result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) - ec = result.error_class - if isinstance(ec, str) and ec.startswith("infra_"): - return False - if result.error is not None: - return False - return True - - -def _resume_field_mismatch( - row: dict[str, Any], - *, - field: str, - expected: str | None, -) -> str | None: - """Return an error message if row[field] is present and disagrees with expected.""" - if expected is None: - return None - raw = row.get(field) - if raw is None or raw == "": - return None # back-compat: older rows without the key pass - # Surface/driver/provider compare case-insensitively; battery/model are exact. - if field in ("surface", "driver", "provider"): - got, want = str(raw).strip().lower(), expected.strip().lower() - else: - got, want = str(raw).strip(), expected.strip() - if got != want: - return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" - return None - - -def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: - """True when a CLI AgentRun stop_reason should be classified as infra_cli. - - ``timeout`` and Claude error subtypes (``error_during_execution``, bare - ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task - failure and stays in the success-rate denominator. - """ - if not stop_reason: - return False - sr = str(stop_reason) - if sr == "timeout": - return True - if sr == "error_max_turns": - return False - if sr == "error" or sr.startswith("error_"): - return True - return False - - -def _timeout_error_message(agent: TaskResult) -> str: - """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" - for note in agent.driver_notes: - if isinstance(note, str) and note.startswith("timeout after"): - return note - return "timeout" - - -def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: - """True for run-header meta lines or any row without a task_id.""" - if row.get("row_type") == "meta": - return True - return row.get("task_id") is None - - -def load_resume_skip_keys( - path: Path, - *, - surface: str, - battery: str | None = None, - model: str | None = None, - driver: str | None = None, - provider: str | None = None, -) -> tuple[set[tuple[str, int]], int, int]: - """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. - - Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` - (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / - battery / model / driver / provider disagrees with the current run (missing keys pass for - back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked - but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. - """ - if not path.is_file(): - return set(), 0, 0 - skip_keys: set[tuple[str, int]] = set() - seen: set[tuple[str, int]] = set() - with path.open(encoding="utf-8") as fh: - for line_no, line in enumerate(fh, start=1): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError as exc: - print( - f"warning: --resume {path}:{line_no}: skipping invalid JSON ({exc})", - file=sys.stderr, - ) - continue - if not isinstance(row, dict): - continue - for field, expected in ( - ("surface", surface), - ("battery", battery), - ("driver", driver), - ("provider", provider), - ): - msg = _resume_field_mismatch(row, field=field, expected=expected) - if msg: - raise SystemExit(msg) - # New tier-aware rows identify the resolved model explicitly. Older - # API rows use requested_model for the resolved ID, while oldest rows - # only have model (which may be provider-reported). - model_row = dict(row) - if model_row.get("resolved_model"): - model_row["model"] = model_row["resolved_model"] - elif model_row.get("requested_model"): - model_row["model"] = model_row["requested_model"] - msg = _resume_field_mismatch(model_row, field="model", expected=model) - if msg: - raise SystemExit(msg) - # Meta / header rows: checked above, not part of resume key set. - if is_meta_or_non_task_row(row): - continue - result = TaskResult.from_row(row) - if not result.task_id: - continue - key = (result.task_id, result.rep) - seen.add(key) - if should_skip_resume_row(result): - skip_keys.add(key) - else: - # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. - skip_keys.discard(key) - n_retry = len(seen - skip_keys) - return skip_keys, len(skip_keys), n_retry - - -def make_run_meta_row( - *, - run_id: str, - surface: str, - battery: str, - model: str | None, - driver: str, - git_sha: str, - provider: str | None = None, - requested_model: str | None = None, - requested_tier: str | None = None, - resolved_model: str | None = None, - ts: str | None = None, -) -> dict[str, Any]: - """Build the single first-line meta record for a new output JSONL.""" - return { - "schema_version": RESULT_SCHEMA_VERSION, - "row_type": "meta", - "run_id": run_id, - "surface": surface, - "battery": battery, - "model": model, - "requested_model": requested_model if requested_model is not None else model, - "requested_tier": requested_tier, - "resolved_model": resolved_model if resolved_model is not None else model, - "driver": driver, - "provider": provider, - "git_sha": git_sha, - "ts": ts or datetime.now(timezone.utc).isoformat(), - } - - -def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: - """Write meta as the first line when the file is missing or empty. Returns True if written.""" - path.parent.mkdir(parents=True, exist_ok=True) - if path.is_file() and path.stat().st_size > 0: - return False - with path.open("w", encoding="utf-8") as fh: - fh.write(json.dumps(meta, default=str) + "\n") - return True - - -async def run_agent_task_via_driver( - *, - driver: Any, - model_id: str | None, - task: dict[str, Any], - ctx: dict[str, Any], - workspace_slug: str, - surface: str = "full", - optimal_tools: set[str] | None = None, - alternate_tools: set[str] | None = None, - server_env: dict[str, str] | None = None, -) -> TaskResult: - """Run one task through the selected AgentDriver.""" - project_name = ctx["project_name"] - system = _system_preamble(workspace_slug, project_name) - prompt = format_task_prompt(task, ctx, strict=True) - optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) - alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) - assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" - - mcp_env = stdio_server_env(surface=surface, extra=server_env) - # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. - agent_run = await asyncio.to_thread( - driver.run_task, - prompt, - mcp_env, - model_id, - MAX_ITERATIONS, - system=system, - cwd=Path(__file__).resolve().parent.parent, - ) - return agent_run_to_task_result( - agent_run, - optimal=optimal, - alternate=alternate, - classify=classify_call, - ) - - -def _base_row( - *, - run_id: str, - git_sha: str, - surface: str, - driver_name: str, - provider: str | None, - model_id: str | None, - model_request: str | None, - requested_tier: str | None, - task: dict[str, Any], - rep: int, - battery: str, - classification: str, -) -> TaskResult: - return TaskResult( - run_id=run_id, - ts=datetime.now(timezone.utc).isoformat(), - git_sha=git_sha, - battery=battery, - surface=surface, - driver=driver_name, - provider=provider, - classification=classification, - model=model_id, - requested_model=model_request, - requested_tier=requested_tier, - resolved_model=model_id, - task_id=str(task["id"]), - author=task_author(task), - rep=rep, - ) - - -async def run_live( - tasks: list[dict[str, Any]], - *, - model_alias: str, - reps: int, - surface: str, - out_path: Path, - driver_name: str = "api", - provider: str = "anthropic", - server_cmd: list[str] | None = None, - server_env: dict[str, str] | None = None, - resume: bool = False, - record_result_payloads: bool = False, - resolved_model_id: str | None = None, -) -> int: - surface = (surface or "full").strip().lower() - external = server_cmd is not None - if not external and surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " - "(or pass --server-cmd for an external surface)", - file=sys.stderr, - ) - return 2 - - driver_name = (driver_name or "api").strip().lower() - if driver_name not in KNOWN_DRIVERS: - print( - f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", - file=sys.stderr, - ) - return 2 - provider = (provider or "anthropic").strip().lower() - is_api_driver = driver_name in ("api", "sdk") - provider_id = provider if is_api_driver else None - model_id = resolved_model_id if resolved_model_id is not None else model_alias - requested_tier = model_alias if model_alias in MODEL_TIERS else None - - run_id = uuid.uuid4().hex - git_sha = _git_sha() - battery = battery_fingerprint(tasks) - out_path.parent.mkdir(parents=True, exist_ok=True) - - resume_skip: set[tuple[str, int]] = set() - if resume: - try: - resume_skip, n_skip, n_retry = load_resume_skip_keys( - out_path, - surface=surface, - battery=battery, - model=model_id, - driver=driver_name, - provider=provider_id, - ) - except SystemExit as e: - print(e, file=sys.stderr) - return 2 - print(f"resume: skipping {n_skip} completed rows, retrying {n_retry}") - - # First line of a new/empty file is a meta header (skipped by loaders). - meta = make_run_meta_row( - run_id=run_id, - surface=surface, - battery=battery, - model=model_id, - requested_model=model_alias, - requested_tier=requested_tier, - resolved_model=model_id, - driver=driver_name, - provider=provider_id, - git_sha=git_sha, - ) - if maybe_write_run_meta(out_path, meta): - print(f"wrote meta header battery={battery} surface={surface}") - - plane, workspace_slug = make_plane_client() - # User chose --driver explicitly: codex live is allowed (they own the quota). - driver_kwargs: dict[str, Any] = {} - if is_api_driver: - driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) - if driver_name == "codex-cli": - driver_kwargs["allow_live"] = True - if not is_api_driver: - driver_kwargs["record_result_payloads"] = record_result_payloads - # --server-cmd must reach every driver; otherwise we - # silently benchmark the wrong server. - if server_cmd is not None: - driver_kwargs["server_command"] = server_cmd - driver = get_driver(driver_name, **driver_kwargs) - - print( - f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} " - f"requested_model={model_alias} resolved_model={model_id} " - f"surface={surface} tasks={[t['id'] for t in tasks]} reps={reps}" - ) - print(f"writing {out_path}") - - async def _run_tasks() -> None: - with out_path.open("a", encoding="utf-8") as fh: - for task in tasks: - if external: - # Foreign tool names have no overlay sets: no skips, no - # mispick classification — success/calls/errors only. - surface_sets = { - "skip": None, - "optimal_tools": set(), - "alternate_tools": set(), - "classification": "external", - } - else: - surface_sets = resolve_surface_tool_sets(task, surface) - for rep in range(reps): - if (task["id"], rep) in resume_skip: - print(f" {task['id']} rep={rep} RESUME_SKIP") - continue - - ctx: dict[str, Any] = {} - row = _base_row( - run_id=run_id, - git_sha=git_sha, - surface=surface, - driver_name=driver_name, - provider=provider_id, - model_id=model_id, - model_request=model_alias, - requested_tier=requested_tier, - task=task, - rep=rep, - battery=battery, - classification=str(surface_sets["classification"]), - ) - try: - # Surface-unsupported tasks: record skip, no seed/agent. - if surface_sets.get("skip"): - reason = surface_sets["skip"] - row.skipped = reason - row.verify_note = reason - print(f" {task['id']} rep={rep} SKIPPED: {reason}") - else: - task_needs = set(task.get("needs") or set()) - # Seed wrap: TaskSkipped → skip; other failures → infra_seed. - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" - print( - f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - if ctx.get("project_name"): - print( - f" orphaned project may remain: {ctx['project_name']}", - file=sys.stderr, - ) - else: - if "bug_type" in task_needs and not ctx.get("bug_type"): - reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" - row.skipped = reason - row.verify_note = reason - print(f" {task['id']} rep={rep} SKIPPED: {reason}") - else: - agent: TaskResult | None = None - # Agent wrap: API failures and CLI failures are infrastructure. - # Contained CLI stops (timeout / error subtypes) return AgentRun. - try: - agent = await run_agent_task_via_driver( - driver=driver, - model_id=model_id, - task=task, - ctx=ctx, - workspace_slug=workspace_slug, - surface=surface, - optimal_tools=surface_sets["optimal_tools"], - alternate_tools=surface_sets["alternate_tools"], - server_env=server_env, - ) - except PromptBindError as exc: - # Empty/missing seed IDs in the prompt — not an agent failure. - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" - print( - f" {task['id']} rep={rep} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - agent = None - except Exception as exc: - if driver_name == "sdk": - agent_err_class = "infra_sdk" - elif is_api_driver: - agent_err_class = "infra_api" - else: - agent_err_class = "infra_cli" - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = agent_err_class - row.verify_note = "" - print( - f" {task['id']} rep={rep} ERROR[{agent_err_class}]: {exc}", - file=sys.stderr, - ) - agent = None - - if agent is not None: - row.apply_agent_result(agent) - # Driver-level requested_model is the resolved ID. - # Restore run-level intent and retain both identities. - row.requested_model = model_alias - row.requested_tier = requested_tier - row.resolved_model = model_id - if external: - # Empty overlay sets would classify every call - # out-of-set; null the counters instead. - row.alternate_calls = None - row.out_of_set_calls = None - - # CLI infra stops: timeout + error subtypes except error_max_turns. - stop_reason = agent.stop_reason - if driver_name.endswith("-cli") and is_infra_cli_stop_reason( - str(stop_reason) if stop_reason is not None else None - ): - row.success = False - row.error_class = "infra_cli" - if stop_reason == "timeout": - row.error = _timeout_error_message(agent) - else: - notes = [n for n in agent.driver_notes if isinstance(n, str)] - detail = "; ".join(notes) if notes else str(stop_reason) - row.error = detail - row.verify_note = "" - print( - f" {task['id']} rep={rep} ERROR[infra_cli]: {row.error}", - file=sys.stderr, - ) - else: - verify = task["verify"] - try: - agent_row = agent.to_row() - ok, note = await verify( - plane, - ctx, - { - "final_text": agent.final_text, - "calls": agent_row["calls"], - }, - ) - row.success = bool(ok) - row.verify_note = note - print( - f" {task['id']} rep={rep} success={ok} " - f"calls={agent.num_calls} note={note!r}" - ) - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={rep} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "task" - row.verify_note = "" - print( - f" {task['id']} rep={rep} ERROR[task]: {exc}", - file=sys.stderr, - ) - except Exception as exc: - # Anything outside seed/driver/verify wraps. - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "task" - row.verify_note = "" - print(f" {task['id']} rep={rep} ERROR[task]: {exc}", file=sys.stderr) - if ctx.get("project_name"): - print( - f" orphaned project may remain: {ctx['project_name']}", - file=sys.stderr, - ) - finally: - try: - teardown(plane, ctx) - except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) - if ctx.get("project_name"): - print(f" orphaned project: {ctx['project_name']}", file=sys.stderr) - - fh.write(json.dumps(row.to_row(), default=str) + "\n") - fh.flush() - - await _run_tasks() - - return 0 - - -async def run_canary( - tasks: list[dict[str, Any]], - *, - surface: str, -) -> int: - """Seed + verify(empty agent) + teardown per task; no driver/model. - - Passes only when every verifier returns falsy ok on a do-nothing agent. - Any ok=True is a broken verifier (false positive). - """ - surface = (surface or "full").strip().lower() - if surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", - file=sys.stderr, - ) - return 2 - - battery = battery_fingerprint(tasks) - plane, _workspace_slug = make_plane_client() - print(f"canary battery={battery} surface={surface} tasks={[t['id'] for t in tasks]}") - - broken: list[str] = [] - verified_count = 0 - empty_agent = {"final_text": "", "calls": []} - - for task in tasks: - surface_sets = resolve_surface_tool_sets(task, surface) - if surface_sets.get("skip"): - print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") - continue - - ctx: dict[str, Any] = {} - task_needs = set(task.get("needs") or set()) - try: - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=ctx) - except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") - continue - if "bug_type" in task_needs and not ctx.get("bug_type"): - reason = ctx.get("bug_type_skip_reason") or "bug_type unavailable" - print(f" {task['id']} SKIPPED: {reason}") - continue - try: - ok, note = await task["verify"](plane, ctx, empty_agent) - except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") - continue - verified_count += 1 - if ok: - broken.append(task["id"]) - print(f" BROKEN VERIFIER: {task['id']} note={note!r}") - else: - print(f" {task['id']} ok=False note={note!r}") - except Exception as exc: - print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) - broken.append(task["id"]) - finally: - try: - teardown(plane, ctx) - except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) - - if broken: - for tid in broken: - print(f"BROKEN VERIFIER: {tid}", file=sys.stderr) - return 1 - if verified_count == 0: - print( - "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", - file=sys.stderr, - ) - return 1 - print(f"canary: all verifiers reject empty agent ({verified_count} verified)") - return 0 - - -__all__ = [ - "KNOWN_SURFACES", - "MAX_ITERATIONS", - "MAX_TOKENS", - "classify_call", - "is_infra_cli_stop_reason", - "is_meta_or_non_task_row", - "load_resume_skip_keys", - "make_run_meta_row", - "maybe_write_run_meta", - "run_agent_task_via_driver", - "run_canary", - "run_live", - "should_skip_resume_row", - "stdio_server_env", -] diff --git a/evals/runner/__init__.py b/evals/runner/__init__.py new file mode 100644 index 00000000..bc6e8945 --- /dev/null +++ b/evals/runner/__init__.py @@ -0,0 +1,32 @@ +"""Live evaluation execution, resume support, metadata, and verifier canary.""" + +from .canary import run_canary +from .live import ( + KNOWN_SURFACES, + MAX_ITERATIONS, + MAX_TOKENS, + classify_call, + is_infra_cli_stop_reason, + run_agent_task_via_driver, + run_live, + stdio_server_env, +) +from .meta import is_meta_or_non_task_row, make_run_meta_row, maybe_write_run_meta +from .resume import load_resume_skip_keys, should_skip_resume_row + +__all__ = [ + "KNOWN_SURFACES", + "MAX_ITERATIONS", + "MAX_TOKENS", + "classify_call", + "is_infra_cli_stop_reason", + "is_meta_or_non_task_row", + "load_resume_skip_keys", + "make_run_meta_row", + "maybe_write_run_meta", + "run_agent_task_via_driver", + "run_canary", + "run_live", + "should_skip_resume_row", + "stdio_server_env", +] diff --git a/evals/runner/canary.py b/evals/runner/canary.py new file mode 100644 index 00000000..fa1f94fd --- /dev/null +++ b/evals/runner/canary.py @@ -0,0 +1,90 @@ +"""Verifier canary execution for evaluation tasks.""" + +from __future__ import annotations + +import sys +import uuid +from typing import Any + +from evals.seed import make_plane_client, seed, teardown +from evals.tasks import TaskSkipped, battery_fingerprint, resolve_surface_tool_sets + +from .live import KNOWN_SURFACES + + +async def run_canary( + tasks: list[dict[str, Any]], + *, + surface: str, +) -> int: + """Seed + verify(empty agent) + teardown per task; no driver/model. + + Passes only when every verifier returns falsy ok on a do-nothing agent. + Any ok=True is a broken verifier (false positive). + """ + surface = (surface or "full").strip().lower() + if surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", + file=sys.stderr, + ) + return 2 + + battery = battery_fingerprint(tasks) + plane, _workspace_slug = make_plane_client() + print(f"canary battery={battery} surface={surface} tasks={[task['id'] for task in tasks]}") + + broken: list[str] = [] + verified_count = 0 + empty_agent = {"final_text": "", "calls": []} + + for task in tasks: + surface_sets = resolve_surface_tool_sets(task, surface) + if surface_sets.get("skip"): + print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") + continue + + context: dict[str, Any] = {} + task_needs = set(task.get("needs") or set()) + try: + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + print(f" {task['id']} SKIPPED: {reason}") + continue + try: + ok, note = await task["verify"](plane, context, empty_agent) + except TaskSkipped as skip: + print(f" {task['id']} SKIPPED: {skip.reason}") + continue + verified_count += 1 + if ok: + broken.append(task["id"]) + print(f" BROKEN VERIFIER: {task['id']} note={note!r}") + else: + print(f" {task['id']} ok=False note={note!r}") + except Exception as exc: + print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) + broken.append(task["id"]) + finally: + try: + teardown(plane, context) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + + if broken: + for task_id in broken: + print(f"BROKEN VERIFIER: {task_id}", file=sys.stderr) + return 1 + if verified_count == 0: + print( + "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", + file=sys.stderr, + ) + return 1 + print(f"canary: all verifiers reject empty agent ({verified_count} verified)") + return 0 diff --git a/evals/runner/live.py b/evals/runner/live.py new file mode 100644 index 00000000..f73cf725 --- /dev/null +++ b/evals/runner/live.py @@ -0,0 +1,519 @@ +"""Live evaluation execution and task result assembly.""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.drivers import KNOWN_DRIVERS, agent_run_to_task_result, get_driver +from evals.drivers.api import MODEL_TIERS +from evals.results import TaskResult +from evals.seed import make_plane_client, seed, teardown +from evals.tasks import ( + PromptBindError, + TaskSkipped, + battery_fingerprint, + format_task_prompt, + resolve_surface_tool_sets, + task_author, +) + +from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision +from .resume import load_resume_skip_keys + +# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). +# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env +# (see plane_mcp.v2.choose_stdio_mcp). +KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) + +MAX_ITERATIONS = 15 +MAX_TOKENS = 8192 + + +def _system_preamble(workspace_slug: str, project_name: str) -> str: + """Keep under 100 words — part of measured context.""" + return ( + f"You are evaluating Plane project management tools. " + f"Workspace slug: {workspace_slug}. Project name: {project_name}. " + f"Complete the task using the available tools, then stop." + ) + + +def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: + if tool in optimal: + return "optimal" + if tool in alternate: + return "alternate" + return "out_of_set" + + +def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6). + + ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the + v2 tool registry. ``surface=full`` leaves the var unset (legacy default). + Other surface names (external servers under benchmark) set nothing; their + selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. + """ + environment: dict[str, str] = {} + if path := os.environ.get("PATH"): + environment["PATH"] = path + if home := os.environ.get("HOME"): + environment["HOME"] = home + environment["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] + environment["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + environment["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if surface == "v2": + environment["PLANE_MCP_SURFACE"] = "v2" + elif surface == "v2-schema": + environment["PLANE_MCP_SURFACE"] = "v2-schema" + if extra: + environment.update(extra) + return environment + + +def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: + """True when a CLI AgentRun stop_reason should be classified as infra_cli. + + ``timeout`` and Claude error subtypes (``error_during_execution``, bare + ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task + failure and stays in the success-rate denominator. + """ + if not stop_reason: + return False + reason = str(stop_reason) + if reason == "timeout": + return True + if reason == "error_max_turns": + return False + if reason == "error" or reason.startswith("error_"): + return True + return False + + +def _timeout_error_message(agent: TaskResult) -> str: + """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" + for note in agent.driver_notes: + if isinstance(note, str) and note.startswith("timeout after"): + return note + return "timeout" + + +async def run_agent_task_via_driver( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + ctx: dict[str, Any], + workspace_slug: str, + surface: str = "full", + optimal_tools: set[str] | None = None, + alternate_tools: set[str] | None = None, + server_env: dict[str, str] | None = None, +) -> TaskResult: + """Run one task through the selected AgentDriver.""" + project_name = ctx["project_name"] + system = _system_preamble(workspace_slug, project_name) + prompt = format_task_prompt(task, ctx, strict=True) + optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) + alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) + assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" + + mcp_env = stdio_server_env(surface=surface, extra=server_env) + # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. + agent_run = await asyncio.to_thread( + driver.run_task, + prompt, + mcp_env, + model_id, + MAX_ITERATIONS, + system=system, + cwd=Path(__file__).resolve().parent.parent.parent, + ) + return agent_run_to_task_result( + agent_run, + optimal=optimal, + alternate=alternate, + classify=classify_call, + ) + + +def _make_task_row( + *, + run_id: str, + git_revision: str, + surface: str, + driver_name: str, + provider: str | None, + model_id: str | None, + model_request: str | None, + requested_tier: str | None, + task: dict[str, Any], + repetition: int, + battery: str, + classification: str, +) -> TaskResult: + return TaskResult( + run_id=run_id, + ts=datetime.now(timezone.utc).isoformat(), + git_sha=git_revision, + battery=battery, + surface=surface, + driver=driver_name, + provider=provider, + classification=classification, + model=model_id, + requested_model=model_request, + requested_tier=requested_tier, + resolved_model=model_id, + task_id=str(task["id"]), + author=task_author(task), + rep=repetition, + ) + + +async def _run_task_repetition( + *, + plane: Any, + driver: Any, + workspace_slug: str, + task: dict[str, Any], + repetition: int, + surface_sets: dict[str, Any], + run_id: str, + git_revision: str, + surface: str, + driver_name: str, + provider_id: str | None, + model_id: str | None, + model_alias: str, + requested_tier: str | None, + battery: str, + is_api_driver: bool, + external: bool, + server_env: dict[str, str] | None, +) -> TaskResult: + """Seed, drive, verify, assemble, and remove one task repetition.""" + context: dict[str, Any] = {} + row = _make_task_row( + run_id=run_id, + git_revision=git_revision, + surface=surface, + driver_name=driver_name, + provider=provider_id, + model_id=model_id, + model_request=model_alias, + requested_tier=requested_tier, + task=task, + repetition=repetition, + battery=battery, + classification=str(surface_sets["classification"]), + ) + try: + # Surface-unsupported tasks: record skip, no seed/agent. + if surface_sets.get("skip"): + reason = surface_sets["skip"] + row.skipped = reason + row.verify_note = reason + print(f" {task['id']} rep={repetition} SKIPPED: {reason}") + else: + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + ) + else: + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + row.skipped = reason + row.verify_note = reason + print(f" {task['id']} rep={repetition} SKIPPED: {reason}") + else: + agent: TaskResult | None = None + # Agent wrap: API failures and CLI failures are infrastructure. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + agent = await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=context, + workspace_slug=workspace_slug, + surface=surface, + optimal_tools=surface_sets["optimal_tools"], + alternate_tools=surface_sets["alternate_tools"], + server_env=server_env, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + agent = None + except Exception as exc: + if driver_name == "sdk": + agent_error_class = "infra_sdk" + elif is_api_driver: + agent_error_class = "infra_api" + else: + agent_error_class = "infra_cli" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = agent_error_class + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", + file=sys.stderr, + ) + agent = None + + if agent is not None: + row.apply_agent_result(agent) + # Driver-level requested_model is the resolved ID. + # Restore run-level intent and retain both identities. + row.requested_model = model_alias + row.requested_tier = requested_tier + row.resolved_model = model_id + if external: + # Empty overlay sets would classify every call + # out-of-set; null the counters instead. + row.alternate_calls = None + row.out_of_set_calls = None + + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.stop_reason + if driver_name.endswith("-cli") and is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): + row.success = False + row.error_class = "infra_cli" + if stop_reason == "timeout": + row.error = _timeout_error_message(agent) + else: + notes = [note for note in agent.driver_notes if isinstance(note, str)] + detail = "; ".join(notes) if notes else str(stop_reason) + row.error = detail + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", + file=sys.stderr, + ) + else: + verify = task["verify"] + try: + agent_row = agent.to_row() + ok, note = await verify( + plane, + context, + { + "final_text": agent.final_text, + "calls": agent_row["calls"], + }, + ) + row.success = bool(ok) + row.verify_note = note + print( + f" {task['id']} rep={repetition} success={ok} " + f"calls={agent.num_calls} note={note!r}" + ) + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[task]: {exc}", + file=sys.stderr, + ) + except Exception as exc: + # Anything outside seed/driver/verify wraps. + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print(f" {task['id']} rep={repetition} ERROR[task]: {exc}", file=sys.stderr) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + ) + finally: + try: + teardown(plane, context) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + if context.get("project_name"): + print(f" orphaned project: {context['project_name']}", file=sys.stderr) + return row + + +async def run_live( + tasks: list[dict[str, Any]], + *, + model_alias: str, + reps: int, + surface: str, + out_path: Path, + driver_name: str = "api", + provider: str = "anthropic", + server_cmd: list[str] | None = None, + server_env: dict[str, str] | None = None, + resume: bool = False, + record_result_payloads: bool = False, + resolved_model_id: str | None = None, +) -> int: + surface = (surface or "full").strip().lower() + external = server_cmd is not None + if not external and surface not in KNOWN_SURFACES: + print( + f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " + "(or pass --server-cmd for an external surface)", + file=sys.stderr, + ) + return 2 + + driver_name = (driver_name or "api").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + provider = (provider or "anthropic").strip().lower() + is_api_driver = driver_name in ("api", "sdk") + provider_id = provider if is_api_driver else None + model_id = resolved_model_id if resolved_model_id is not None else model_alias + requested_tier = model_alias if model_alias in MODEL_TIERS else None + + run_id = uuid.uuid4().hex + git_revision = read_git_revision() + battery = battery_fingerprint(tasks) + out_path.parent.mkdir(parents=True, exist_ok=True) + + resume_skip: set[tuple[str, int]] = set() + if resume: + try: + resume_skip, skip_count, retry_count = load_resume_skip_keys( + out_path, + surface=surface, + battery=battery, + model=model_id, + driver=driver_name, + provider=provider_id, + ) + except SystemExit as exc: + print(exc, file=sys.stderr) + return 2 + print(f"resume: skipping {skip_count} completed rows, retrying {retry_count}") + + # First line of a new/empty file is a meta header (skipped by loaders). + meta = make_run_meta_row( + run_id=run_id, + surface=surface, + battery=battery, + model=model_id, + requested_model=model_alias, + requested_tier=requested_tier, + resolved_model=model_id, + driver=driver_name, + provider=provider_id, + git_sha=git_revision, + ) + if maybe_write_run_meta(out_path, meta): + print(f"wrote meta header battery={battery} surface={surface}") + + plane, workspace_slug = make_plane_client() + # User chose --driver explicitly: codex live is allowed (they own the quota). + driver_kwargs: dict[str, Any] = {} + if is_api_driver: + driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) + if driver_name == "codex-cli": + driver_kwargs["allow_live"] = True + if not is_api_driver: + driver_kwargs["record_result_payloads"] = record_result_payloads + # --server-cmd must reach every driver; otherwise we + # silently benchmark the wrong server. + if server_cmd is not None: + driver_kwargs["server_command"] = server_cmd + driver = get_driver(driver_name, **driver_kwargs) + + print( + f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} " + f"requested_model={model_alias} resolved_model={model_id} " + f"surface={surface} tasks={[task['id'] for task in tasks]} reps={reps}" + ) + print(f"writing {out_path}") + + with out_path.open("a", encoding="utf-8") as file: + for task in tasks: + if external: + # Foreign tool names have no overlay sets: no skips, no + # mispick classification — success/calls/errors only. + surface_sets = { + "skip": None, + "optimal_tools": set(), + "alternate_tools": set(), + "classification": "external", + } + else: + surface_sets = resolve_surface_tool_sets(task, surface) + for repetition in range(reps): + if (task["id"], repetition) in resume_skip: + print(f" {task['id']} rep={repetition} RESUME_SKIP") + continue + row = await _run_task_repetition( + plane=plane, + driver=driver, + workspace_slug=workspace_slug, + task=task, + repetition=repetition, + surface_sets=surface_sets, + run_id=run_id, + git_revision=git_revision, + surface=surface, + driver_name=driver_name, + provider_id=provider_id, + model_id=model_id, + model_alias=model_alias, + requested_tier=requested_tier, + battery=battery, + is_api_driver=is_api_driver, + external=external, + server_env=server_env, + ) + file.write(json.dumps(row.to_row(), default=str) + "\n") + file.flush() + + return 0 diff --git a/evals/runner/meta.py b/evals/runner/meta.py new file mode 100644 index 00000000..9b395bdd --- /dev/null +++ b/evals/runner/meta.py @@ -0,0 +1,75 @@ +"""Run metadata and repository provenance for evaluation results.""" + +from __future__ import annotations + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.results import RESULT_SCHEMA_VERSION + + +def read_git_revision() -> str: + try: + return ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + cwd=Path(__file__).resolve().parent.parent.parent, + ) + .decode() + .strip() + ) + except Exception: + return "unknown" + + +def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: + """True for run-header meta lines or any row without a task_id.""" + if row.get("row_type") == "meta": + return True + return row.get("task_id") is None + + +def make_run_meta_row( + *, + run_id: str, + surface: str, + battery: str, + model: str | None, + driver: str, + git_sha: str, + provider: str | None = None, + requested_model: str | None = None, + requested_tier: str | None = None, + resolved_model: str | None = None, + ts: str | None = None, +) -> dict[str, Any]: + """Build the single first-line meta record for a new output JSONL.""" + return { + "schema_version": RESULT_SCHEMA_VERSION, + "row_type": "meta", + "run_id": run_id, + "surface": surface, + "battery": battery, + "model": model, + "requested_model": requested_model if requested_model is not None else model, + "requested_tier": requested_tier, + "resolved_model": resolved_model if resolved_model is not None else model, + "driver": driver, + "provider": provider, + "git_sha": git_sha, + "ts": ts or datetime.now(timezone.utc).isoformat(), + } + + +def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: + """Write meta as the first line when the file is missing or empty. Returns True if written.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file() and path.stat().st_size > 0: + return False + with path.open("w", encoding="utf-8") as file: + file.write(json.dumps(meta, default=str) + "\n") + return True diff --git a/evals/runner/resume.py b/evals/runner/resume.py new file mode 100644 index 00000000..bee70ed1 --- /dev/null +++ b/evals/runner/resume.py @@ -0,0 +1,124 @@ +"""Resume decisions for evaluation result files.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.results import TaskResult + +from .meta import is_meta_or_non_task_row + + +def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: + """Return True if a prior row is a completed result that resume should skip. + + Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. + Rows with ``skipped`` set are treated as complete and are not retried (intentional: + surface/plan skips are stable outcomes, not infra failures). + Pure function — unit-tested without the live battery. + """ + result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) + error_class = result.error_class + if isinstance(error_class, str) and error_class.startswith("infra_"): + return False + if result.error is not None: + return False + return True + + +def _resume_field_mismatch( + row: dict[str, Any], + *, + field: str, + expected: str | None, +) -> str | None: + """Return an error message if row[field] is present and disagrees with expected.""" + if expected is None: + return None + raw = row.get(field) + if raw is None or raw == "": + return None # back-compat: older rows without the key pass + # Surface/driver/provider compare case-insensitively; battery/model are exact. + if field in ("surface", "driver", "provider"): + received, wanted = str(raw).strip().lower(), expected.strip().lower() + else: + received, wanted = str(raw).strip(), expected.strip() + if received != wanted: + return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" + return None + + +def load_resume_skip_keys( + path: Path, + *, + surface: str, + battery: str | None = None, + model: str | None = None, + driver: str | None = None, + provider: str | None = None, +) -> tuple[set[tuple[str, int]], int, int]: + """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. + + Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` + (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / + battery / model / driver / provider disagrees with the current run (missing keys pass for + back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked + but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. + """ + if not path.is_file(): + return set(), 0, 0 + skip_keys: set[tuple[str, int]] = set() + seen: set[tuple[str, int]] = set() + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: --resume {path}:{line_number}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict): + continue + for field, expected in ( + ("surface", surface), + ("battery", battery), + ("driver", driver), + ("provider", provider), + ): + message = _resume_field_mismatch(row, field=field, expected=expected) + if message: + raise SystemExit(message) + # New tier-aware rows identify the resolved model explicitly. Older + # API rows use requested_model for the resolved ID, while oldest rows + # only have model (which may be provider-reported). + model_row = dict(row) + if model_row.get("resolved_model"): + model_row["model"] = model_row["resolved_model"] + elif model_row.get("requested_model"): + model_row["model"] = model_row["requested_model"] + message = _resume_field_mismatch(model_row, field="model", expected=model) + if message: + raise SystemExit(message) + # Meta / header rows: checked above, not part of resume key set. + if is_meta_or_non_task_row(row): + continue + result = TaskResult.from_row(row) + if not result.task_id: + continue + key = (result.task_id, result.rep) + seen.add(key) + if should_skip_resume_row(result): + skip_keys.add(key) + else: + # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. + skip_keys.discard(key) + retry_count = len(seen - skip_keys) + return skip_keys, len(skip_keys), retry_count diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py index 1917b7bb..882f1077 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/test_evals_api_driver.py @@ -27,7 +27,7 @@ resolve_backend_model, unregister_backend, ) -from evals.token_counting import estimate_result_tokens +from evals.drivers.token_counting import estimate_result_tokens class FakeBackend: diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index db66672e..394d0657 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -32,8 +32,8 @@ strip_mcp_prefix, write_claude_mcp_config, ) +from evals.drivers.token_counting import estimate_result_tokens from evals.run import classify_call, parse_args, stdio_server_env -from evals.token_counting import estimate_result_tokens # --------------------------------------------------------------------------- # Fixtures (constructed — never captured from live CLIs) diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 6d9e9cfe..2e730d0e 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -14,7 +14,6 @@ from evals import report as report_mod from evals import run as run_mod -from evals import runner as runner_mod from evals import seed as seed_mod from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize @@ -26,6 +25,8 @@ run_live, should_skip_resume_row, ) +from evals.runner import canary as runner_canary +from evals.runner import live as runner_live from evals.seed import create_project_with_identifier_retry, is_identifier_collision from evals.tasks import battery_fingerprint, task_author @@ -194,8 +195,8 @@ def test_load_resume_skip_keys_missing_file(tmp_path: Path): def test_parse_args_resume_and_canary(): - a = run_mod.parse_args(["--resume", "evals/results/x.jsonl", "--dry-run"]) - assert a.resume == "evals/results/x.jsonl" + a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) + assert a.resume == "evals/output/x.jsonl" b = run_mod.parse_args(["--canary", "--tasks", "R1"]) assert b.canary is True @@ -214,14 +215,14 @@ def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) def boom_seed(plane, run_id, needs, ctx): ctx["project_name"] = "EVAL deadbeef" raise HttpError("identifier already taken", 409) - monkeypatch.setattr(runner_mod, "seed", boom_seed) - monkeypatch.setattr(runner_mod, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "seed", boom_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) task = { "id": "T1", @@ -267,13 +268,13 @@ def boom_seed(plane, run_id, needs, ctx): def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) def ok_seed(plane, run_id, needs, ctx): ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) - monkeypatch.setattr(runner_mod, "seed", ok_seed) - monkeypatch.setattr(runner_mod, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) class BoomDriver: name = "claude-cli" @@ -281,7 +282,7 @@ class BoomDriver: def run_task(self, *args, **kwargs): raise RuntimeError("claude cli failed: json_parse_failed") - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: BoomDriver()) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) task = { "id": "T2", @@ -316,9 +317,9 @@ def test_run_live_timeout_agent_is_infra_cli(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) class TimeoutDriver: name = "claude-cli" @@ -332,7 +333,7 @@ def run_task(self, *args, **kwargs): notes=["timeout after 900s"], ) - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: TimeoutDriver()) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) verify_calls: list[Any] = [] @@ -380,9 +381,9 @@ def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatc """exit 1 + parseable JSON subtype error_during_execution → infra_cli; verify not called.""" out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) payload = { "type": "result", @@ -397,7 +398,7 @@ def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatc def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) verify_calls: list[Any] = [] @@ -437,9 +438,9 @@ def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): """exit 1 + subtype error_max_turns stays in the task denominator (not infra_cli).""" out = tmp_path / "rows.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) payload = { "type": "result", @@ -454,7 +455,7 @@ def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) verify_calls: list[Any] = [] @@ -818,9 +819,11 @@ def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): def test_canary_detects_broken_verifier(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) async def always_ok(plane, ctx, run): return True, "false positive" @@ -854,9 +857,11 @@ async def correctly_fails(plane, ctx, run): def test_canary_passes_when_all_verifiers_reject(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) async def reject(plane, ctx, run): assert run == {"final_text": "", "calls": []} @@ -879,11 +884,11 @@ async def reject(plane, ctx, run): def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_mod, "seed", lambda *a, **k: None) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_canary, "seed", lambda *a, **k: None) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) monkeypatch.setattr( - runner_mod, + runner_canary, "resolve_surface_tool_sets", lambda task, surface: { "skip": "unsupported on surface", @@ -915,7 +920,7 @@ def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path: Path, monkeypatch): out = tmp_path / "multi.jsonl" fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) seed_ids: list[str] = [] teardown_projects: list[str] = [] @@ -929,10 +934,10 @@ def record_teardown(plane, ctx): async def fake_agent(**kwargs): return TaskResult(final_text="done", stop_reason="end_turn") - monkeypatch.setattr(runner_mod, "seed", fresh_seed) - monkeypatch.setattr(runner_mod, "teardown", record_teardown) - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kwargs: object()) - monkeypatch.setattr(runner_mod, "run_agent_task_via_driver", fake_agent) + monkeypatch.setattr(runner_live, "seed", fresh_seed) + monkeypatch.setattr(runner_live, "teardown", record_teardown) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) + monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) async def verify_ok(plane, ctx, run): return True, "ok" @@ -998,7 +1003,7 @@ def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypat out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") fake_plane = MagicMock() - monkeypatch.setattr(runner_mod, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) seed_calls: list[str] = [] def ok_seed(plane, run_id, needs, ctx): @@ -1006,8 +1011,8 @@ def ok_seed(plane, run_id, needs, ctx): ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) seed_calls.append(run_id) - monkeypatch.setattr(runner_mod, "seed", ok_seed) - monkeypatch.setattr(runner_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) class OkDriver: name = "claude-cli" @@ -1020,7 +1025,7 @@ def run_task(self, *args, **kwargs): stopped_reason="end_turn", ) - monkeypatch.setattr(runner_mod, "get_driver", lambda name, **kw: OkDriver()) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: OkDriver()) async def verify_ok(plane, ctx, run): return True, "ok" diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index 4c927269..0a098338 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -30,6 +30,7 @@ write_opencode_mcp_config, ) from evals.drivers.cli import CliDriver, CliLaunch, CliOutput +from evals.drivers.token_counting import estimate_result_tokens from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -42,7 +43,6 @@ from evals.proxy import main as proxy_main from evals.run import main as eval_main from evals.run import resolve_model_for_driver -from evals.token_counting import estimate_result_tokens REPO = Path(__file__).resolve().parent.parent @@ -1833,7 +1833,7 @@ def fake_run(cmd, **kwargs): def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): """--server-cmd must not be Claude-only.""" - from evals import runner as run_mod + from evals.runner import live as run_mod captured: dict = {} diff --git a/tests/test_evals_surface.py b/tests/test_evals_surface.py index b8a7a945..5f85c82e 100644 --- a/tests/test_evals_surface.py +++ b/tests/test_evals_surface.py @@ -114,7 +114,7 @@ def test_classify_uses_resolved_sets(): def test_skip_path_no_network(monkeypatch): """Unsupported surface skip must not call seed/teardown/agent.""" - from evals import runner as run_mod + from evals.runner import live as run_mod seeded = [] torn = [] From ba60db7c069666a000bf54efdc8ef9723079e534 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 16:18:21 +0530 Subject: [PATCH 16/93] Drop the abandoned v2 surface vocabulary from the eval harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness once measured three tool surfaces served by one server: full, v2, and v2-schema. The v2 work is set aside, and no server this tree can launch reads PLANE_MCP_SURFACE — so `--surface v2` ran the legacy server under a v2 label. Every real comparison used --server-cmd, where overlay-based tool-choice classification is disabled anyway, which left the whole overlay mechanism dead. A run now says what to call its column (--label, free-form) and what server to launch (--server-cmd / --server-env). Those were one flag before. - `--surface` becomes `--label`; rows carry `label`, and `classification` becomes `server`, which is `local` or `external` and says whether the mispick counters mean anything. - Task overlays, resolve_surface_tool_sets, and PLANE_MCP_SURFACE are gone. Tasks keep one optimal/alternate set each; no prompt, verifier, fixture, or tool set changed. - The `sdk` driver alias and its infra_sdk error class are gone: no result row on this machine ever used either. - evals/run.py was a facade holding a duplicate run_live that re-resolved the model. The command is now `python -m evals`. A skipped task keeps its own test now that overlays no longer supply one: a seed that raises TaskSkipped must record the skip, leave the success denominator empty, and still tear down. Verified live against a local Plane: the canary rejects a do-nothing agent on all 33 verifiers, and R1/W1 pass through the real driver with the workspace left empty. --- evals/DESIGN.md | 18 +- evals/README.md | 56 ++-- evals/__main__.py | 6 + evals/cli.py | 50 ++-- evals/drivers/__init__.py | 6 +- evals/drivers/base.py | 2 +- evals/listing.py | 42 +-- evals/report/command.py | 6 +- evals/report/load.py | 12 +- evals/report/table.py | 14 +- evals/results.py | 14 +- evals/run.py | 109 -------- evals/runner/__init__.py | 2 - evals/runner/canary.py | 25 +- evals/runner/live.py | 310 +++++++++------------ evals/runner/meta.py | 6 +- evals/runner/resume.py | 22 +- evals/tasks/__init__.py | 93 +------ evals/tasks/cross.py | 28 -- evals/tasks/debias.py | 84 ------ evals/tasks/read.py | 99 ------- evals/tasks/schema.py | 107 ------- evals/tasks/write.py | 123 -------- tests/fixtures/evals_historical_rows.jsonl | 4 +- tests/test_evals_catalog.py | 113 +------- tests/test_evals_drivers.py | 28 +- tests/test_evals_hardening.py | 191 ++++++++----- tests/test_evals_proxy.py | 9 +- tests/test_evals_report_ops.py | 131 ++++----- tests/test_evals_surface.py | 155 ----------- 30 files changed, 460 insertions(+), 1405 deletions(-) create mode 100644 evals/__main__.py delete mode 100644 evals/run.py delete mode 100644 tests/test_evals_surface.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 07d5afc9..edfbc970 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -54,15 +54,15 @@ one run is representative. Client-local tools such as shell or tool-search helpers are retained separately as `client_tool_calls`; they do not count as Plane calls. For an external server launched with -`--server-cmd`, call counts still apply, but the runner marks classification as `external` +`--server-cmd`, call counts still apply, but the runner marks the row server as `external` and clears the row-level alternate/out-of-set counters because the catalog has no authoritative sets for foreign tool names. ### Mispicks Every Plane call on a catalogued surface is classified by tool name as `optimal`, -`alternate`, or `out_of_set`. The task owns disjoint optimal and alternate sets, including -surface-specific overlays where present. The headline mispick rate is: +`alternate`, or `out_of_set`. The task owns disjoint optimal and alternate sets. The +headline mispick rate is: ```text (alternate calls + out-of-set calls) / all Plane calls @@ -126,8 +126,7 @@ All five driver implementations satisfy `AgentDriver.run_task(...) -> AgentRun`: - `AntigravityCliDriver` - `OpencodeCliDriver` -`sdk` is a legacy CLI alias for `ApiDriver`, not a sixth implementation. The runner has one -path for all drivers: it supplies a prompt and MCP environment, receives a normalized +The runner has one path for all drivers: it supplies a prompt and MCP environment, receives a normalized `AgentRun`, maps it to the common row shape, and invokes the task verifier. The API implementation has one further seam. `ApiDriver` owns provider-independent policy: @@ -180,11 +179,10 @@ seed -> drive -> verify -> teardown -> append row The row is assembled as the task progresses; teardown runs in `finally` before that row is appended. Workspace-scoped fixture objects are tracked separately from the project. A fresh stdio server is launched for each driven task. The server environment is built from `PATH`, -`HOME`, the three Plane connection values, the selected built-in surface label when -applicable, and explicit `--server-env` additions; unrelated parent environment variables +`HOME`, the three Plane connection values, and explicit `--server-env` additions; unrelated parent environment variables are not inherited. -The first line of a new result file is a meta row containing the run identity, surface, +The first line of a new result file is a meta row containing the run identity, label, server, battery, requested model/tier, resolved model, driver, provider, and Git SHA. Resume checks those identities, skips completed task/repetition keys, and reruns rows that contain recorded errors. Result rows preserve the common fields consumed by `evals.report` and existing JSONL readers. @@ -194,13 +192,13 @@ rows preserve the common fields consumed by `evals.report` and existing JSONL re ```text evals/ cli.py argparse, command dispatch, and model-tier resolution + __main__.py command entry point for python -m evals runner/ __init__.py public execution API live.py live lifecycle and row assembly resume.py resume skip and mismatch checks meta.py run metadata and repository provenance canary.py empty-agent verifier canary - run.py compatibility entry point for python -m evals.run tasks/ __init__.py ordered catalog assembly and public task API common.py prompt binding, matchers, and shared API lookups @@ -230,5 +228,5 @@ evals/ ``` The stable import and command surfaces are intentional: `from evals.tasks import ...`, -`from evals.drivers import ...`, and `python -m evals.run` remain the public boundaries even +`from evals.drivers import ...`, and `python -m evals` remain the public boundaries even though their implementations are split across packages and focused modules. diff --git a/evals/README.md b/evals/README.md index 2729f5fb..fd145a5d 100644 --- a/evals/README.md +++ b/evals/README.md @@ -43,45 +43,36 @@ agent's final text. ```bash # Provider-neutral API loop (default provider: Anthropic) -.venv/bin/python -m evals.run --driver api --provider anthropic --model standard \ - --surface full --out evals/output/api.jsonl +.venv/bin/python -m evals --driver api --provider anthropic --model standard \ + --label local --out evals/output/api.jsonl # Everything, one surface (free-form model IDs pass through to the CLI) -.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ - --surface full --out evals/output/legacy.jsonl +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --out evals/output/local.jsonl # A few tasks while iterating -.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ - --surface full --tasks W5,W8 --out evals/output/spot.jsonl +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --tasks W5,W8 --out evals/output/spot.jsonl # Someone else's server (a PR branch, another repo) — "external mode" -.venv/bin/python -m evals.run --driver codex-cli --model YOUR_CODEX_MODEL_ID \ - --surface their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ - --server-env PLANE_MCP_TOOLS_VERSION=v2 --out evals/output/their-pr.jsonl +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ + --server-env KEY=VALUE --out evals/output/their-pr.jsonl ``` Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed or -skipped `(task, rep)` pairs and retry rows with recorded errors), `--list` / `--dry-run` +skipped `(task, rep, label)` keys and retry rows with recorded errors), `--list` / `--dry-run` (no network). -**External mode** (`--server-cmd`) runs every task with no surface-based skips and records -`classification: "external"`. Foreign tool names have no catalogued optimal/alternate sets, +**External mode** (`--server-cmd`) records `server: "external"`. Foreign tool names have no catalogued optimal/alternate sets, so use success, call counts, and errors for those rows; their mispick values are not comparable to catalogued surfaces. -**On surfaces.** Without `--server-cmd`, `full` launches this repo's server with no surface -variable. The `v2` and `v2-schema` labels set `PLANE_MCP_SURFACE`, and the task catalog has -matching `surface_tools` overlays. This tree's `plane_mcp` server does not read that variable, -so selecting either label here would run the same server under a misleading label. Use -`--server-cmd` to launch a server that actually implements another surface; external rows -intentionally do not use this catalog's tool-choice classification. - ### Drivers | Driver | Backend | Notes | |---|---|---| | `api` | Owned API + MCP loop | Provider-neutral; tiers resolve for `--provider anthropic` (default) or `openai` | -| `sdk` | Alias for `api` | Retained for old commands/result pipelines | | `codex-cli` | OpenAI Codex CLI | `standard` and `fast` resolve to verified GPT-5.6 IDs | | `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku` | | `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; runs under a synthetic HOME so its MCP config is ours, not yours | @@ -94,8 +85,8 @@ battery, and `fast`, the lower-cost option. Resolution is scoped to both driver | Driver | Provider | `standard` | `fast` | |---|---|---|---| -| `api` / `sdk` | Anthropic | `claude-sonnet-5` | `claude-haiku-4-5` | -| `api` / `sdk` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | +| `api` | Anthropic | `claude-sonnet-5` | `claude-haiku-4-5` | +| `api` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | | `claude-cli` | Anthropic | `sonnet` | `haiku` | | `codex-cli` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | | `antigravity-cli` | Google | `gemini-3.6-flash-high` | `gemini-3.6-flash-low` | @@ -133,12 +124,12 @@ habit; use it only when the more sensitive, larger sidecar is justified. ### Reading results ```bash -.venv/bin/python -m evals.report evals/output/legacy.jsonl # one surface +.venv/bin/python -m evals.report evals/output/local.jsonl # one surface .venv/bin/python -m evals.report --table evals/output/*.jsonl # side by side .venv/bin/python -m evals.report --table --markdown evals/output/*.jsonl # for a PR ``` -Rows are deduped latest-wins per `(task_id, rep, surface)`, so a re-run of a single task +Rows are deduped latest-wins per `(task_id, rep, label)`, so a re-run of a single task supersedes its earlier row in the same file. Skipped tasks are excluded from success denominators, as are rows with recorded errors. Result-token columns use `~` for estimates, `*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not @@ -164,8 +155,8 @@ Tasks that touch **workspace-scoped** fixtures (release tags, customer propertie if two runs share a workspace. Give each concurrent run its own workspace: ```bash -EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --surface full --out evals/output/legacy.jsonl & -EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --surface their-pr --out evals/output/their-pr.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --label local --out evals/output/local.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --label their-pr --out evals/output/their-pr.jsonl & wait ``` @@ -195,12 +186,6 @@ A task is a dict: "optimal_calls": 3, "optimal_tools": {"list_cycles", "complete_cycle"}, # scored as optimal picks "alternate_tools": {"list_projects"}, # acceptable, not optimal - "surface_tools": { - "v2": { - "optimal_tools": {"close_cycle"}, - "alternate_tools": {"list_cycles"}, - } - }, "needs": {"items", "cycles"}, # fixtures to seed "verify": verify_w11, } @@ -211,9 +196,6 @@ A task is a dict: `leave_cycles_worklogs_off`. Each task gets its own freshly seeded project, so fixture variants (e.g. `cycles_open_past`) don't leak between tasks. -A `surface_tools` overlay may set `"expected_skip": True` to declare that a surface -genuinely cannot do the task. That is reported as a capability gap, not a failure. - ### Writing a verifier Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)`. Keep a new task @@ -235,10 +217,10 @@ shape. Then prove the verifier can fail: ```bash -.venv/bin/python -m evals.run --canary --surface full +.venv/bin/python -m evals --canary --label local ``` -The canary seeds each surface-eligible task, calls its verifier with an **empty** agent +The canary seeds each task, calls its verifier with an **empty** agent result, and exits non-zero if any verifier passes a do-nothing agent. Run it after touching tasks, fixtures, or verifiers. diff --git a/evals/__main__.py b/evals/__main__.py new file mode 100644 index 00000000..97248a5d --- /dev/null +++ b/evals/__main__.py @@ -0,0 +1,6 @@ +"""Command entry point: python -m evals""" + +from evals.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/cli.py b/evals/cli.py index c4211766..dbfa0b8d 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -1,8 +1,4 @@ -"""Command-line wiring and model resolution for the eval harness. - -The stable process entry point remains ``python -m evals.run``; that module -delegates here. -""" +"""Command-line wiring and model resolution for the eval harness.""" from __future__ import annotations @@ -22,7 +18,7 @@ backend_model_aliases, resolve_backend_model, ) -from evals.runner import KNOWN_SURFACES, run_canary, run_live +from evals.runner import run_canary, run_live from evals.seed import seed_plan from evals.tasks import TASKS, format_task_prompt, get_tasks @@ -66,7 +62,7 @@ def resolve_model_for_driver(driver_name: str, model: str, *, provider: str | No vendor aliases and qualified provider/model IDs, is passed through exactly. """ key = (driver_name or "api").strip().lower() - if key in ("api", "sdk"): + if key == "api": return resolve_backend_model(provider or "anthropic", model) if model not in MODEL_TIERS: return model @@ -105,23 +101,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) p.add_argument("--reps", type=int, default=1, help="Repetitions per task") p.add_argument( - "--surface", + "--label", type=str, - default="full", - help=( - "Tool surface: 'full' (legacy 177 tools), 'v2', or 'v2-schema'. " - "With --server-cmd it is a free-form label for the external surface." - ), + default="local", + help="Column label for this run in reports (default: local).", ) p.add_argument( "--server-cmd", type=str, default=None, help=( - "External MCP stdio server launch command (shlex-split), e.g. " - "'/path/venv/bin/python -m plane_mcp stdio --v2'. Enables external mode: " - "all tasks run (no surface skips) and mispick classification is disabled " - "(the foreign tool names have no overlay sets)." + "External MCP stdio server launch command (shlex-split). Enables external " + "mode, where foreign tool names make mispick classification unavailable." ), ) p.add_argument( @@ -137,8 +128,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="api", choices=sorted(KNOWN_DRIVERS), help=( - "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli " - "('sdk' is an alias for 'api'). Not required for --canary." + "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli. Not required for --canary." ), ) p.add_argument( @@ -146,7 +136,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=str, default="anthropic", choices=sorted(KNOWN_API_PROVIDERS), - help="Model API provider for --driver api/sdk (default: anthropic).", + help="Model API provider for --driver api (default: anthropic).", ) p.add_argument( "--record-result-payloads", @@ -163,8 +153,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=None, metavar="OUT.jsonl", help=( - "Resume into an existing JSONL (also the --out target). Skip (task_id, rep) " - "pairs that already completed; re-run rows with infra_ error_class or non-null error." + "Resume into an existing JSONL (also the --out target). Skip " + "(task_id, rep, label) keys that already completed; re-run rows with infra_ " + "error_class or non-null error." ), ) p.add_argument( @@ -237,20 +228,13 @@ def main(argv: list[str] | None = None) -> int: print("error: --reps must be at least 1", file=sys.stderr) return 2 - surface = (args.surface or "full").strip().lower() + label = (args.label or "local").strip() or "local" server_cmd: list[str] | None = None if args.server_cmd: server_cmd = shlex.split(args.server_cmd) if not server_cmd: print("error: --server-cmd is empty", file=sys.stderr) return 2 - elif surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " - "(or pass --server-cmd for an external surface)", - file=sys.stderr, - ) - return 2 server_env: dict[str, str] = {} for pair in args.server_env: @@ -262,7 +246,7 @@ def main(argv: list[str] | None = None) -> int: # Canary: live env only — no driver/model required. if args.canary: - return asyncio.run(run_canary(tasks, surface=surface)) + return asyncio.run(run_canary(tasks, label=label)) driver_name = (getattr(args, "driver", None) or "api").strip().lower() if driver_name not in KNOWN_DRIVERS: @@ -283,7 +267,7 @@ def main(argv: list[str] | None = None) -> int: model_id = resolve_model_for_driver( driver_name, args.model, - provider=args.provider if driver_name in ("api", "sdk") else None, + provider=args.provider if driver_name == "api" else None, ) except ValueError as exc: print(f"error: {exc}", file=sys.stderr) @@ -293,7 +277,7 @@ def main(argv: list[str] | None = None) -> int: tasks, model_alias=args.model, reps=args.reps, - surface=surface, + label=label, out_path=out, driver_name=driver_name, provider=args.provider, diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 647c5e46..daf2c4ee 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -87,13 +87,13 @@ # Registry # --------------------------------------------------------------------------- -KNOWN_DRIVERS = frozenset({"api", "sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) +KNOWN_DRIVERS = frozenset({"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) def get_driver(name: str, **kwargs: Any) -> AgentDriver: - """Return a driver instance; ``sdk`` is a legacy alias for ``api``.""" + """Return a driver instance.""" key = (name or "api").strip().lower() - if key in ("api", "sdk"): + if key == "api": return ApiDriver(**kwargs) if key == "claude-cli": return ClaudeCliDriver(**kwargs) diff --git a/evals/drivers/base.py b/evals/drivers/base.py index d7f7bd53..8f575a26 100644 --- a/evals/drivers/base.py +++ b/evals/drivers/base.py @@ -63,7 +63,7 @@ class AgentRun: class AgentDriver(Protocol): - """Pluggable agent backend for evals.run.""" + """Pluggable agent backend for the evaluation harness.""" name: str diff --git a/evals/listing.py b/evals/listing.py index d1200a95..7eb06f97 100644 --- a/evals/listing.py +++ b/evals/listing.py @@ -1,8 +1,7 @@ """Measure MCP tool listing size (tool count + cl100k tokens). Usage: - python -m evals.listing --surface v2 - python -m evals.listing --surface full + python -m evals.listing --label local python -m evals.listing --server-cmd '/path/bin/python -m plane_mcp stdio' --server-env KEY=VAL Reports: tool count, wire listing tokens (incl. outputSchema), model-facing tokens @@ -24,9 +23,8 @@ from pathlib import Path from typing import Any -from evals.run import stdio_server_env +from evals.runner.live import stdio_server_env -KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) REPO_ROOT = Path(__file__).resolve().parent.parent @@ -112,11 +110,11 @@ def count_tool_tokens( return rows, total_wire, total_model -def _listing_stdio_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: - """Build MCP stdio env from EVAL_* credentials via the shared run.py helper.""" +def _listing_stdio_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from EVAL_* credentials via the shared runner helper.""" if not os.environ.get("EVAL_PLANE_API_KEY") or not os.environ.get("EVAL_PLANE_WORKSPACE_SLUG"): raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for listing measurement") - return stdio_server_env(surface=surface, extra=extra) + return stdio_server_env(extra=extra) async def list_tools_from_stdio( @@ -146,19 +144,16 @@ async def list_tools_from_stdio( def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p = argparse.ArgumentParser(description="Measure MCP tool listing tokens (cl100k)") p.add_argument( - "--surface", + "--label", type=str, - default=None, - help=( - "Tool surface: full | v2 | v2-schema. Default full when not using --server-cmd; " - "with --server-cmd, default label is 'external' (or pass a free-form label)." - ), + default="local", + help="Label printed with the listing measurement (default: local).", ) p.add_argument( "--server-cmd", type=str, default=None, - help="External MCP stdio launch command (shlex-split); free-form surface label", + help="External MCP stdio launch command (shlex-split)", ) p.add_argument( "--server-env", @@ -187,27 +182,18 @@ def main(argv: list[str] | None = None) -> int: print("error: --server-cmd is empty", file=sys.stderr) return 2 command, cmd_args = parts[0], parts[1:] - # Never label external runs as "full" — default is "external". - surface_label = (args.surface or "external").strip() or "external" + label = (args.label or "local").strip() or "local" try: - # surface="full" leaves PLANE_MCP_SURFACE unset; extra may set foreign vars. - env = _listing_stdio_env(surface="full", extra=extra or None) + env = _listing_stdio_env(extra=extra or None) except RuntimeError as exc: print(f"error: {exc}", file=sys.stderr) return 2 else: - surface = (args.surface or "full").strip().lower() - if surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", - file=sys.stderr, - ) - return 2 command = sys.executable cmd_args = ["-m", "plane_mcp", "stdio"] - surface_label = surface + label = (args.label or "local").strip() or "local" try: - env = _listing_stdio_env(surface=surface, extra=extra or None) + env = _listing_stdio_env(extra=extra or None) except RuntimeError as exc: print(f"error: {exc}", file=sys.stderr) return 2 @@ -229,7 +215,7 @@ def main(argv: list[str] | None = None) -> int: with_out = sum(1 for r in rows if r.has_output_schema) print( - f"surface={surface_label} tools={len(rows)} " + f"label={label} tools={len(rows)} " f"listing_tokens_cl100k={total_wire} " f"model_facing(no_outputSchema)={total_model} " f"tools_with_outputSchema={with_out}" diff --git a/evals/report/command.py b/evals/report/command.py index 601189a7..e1db86ae 100644 --- a/evals/report/command.py +++ b/evals/report/command.py @@ -30,7 +30,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--table", action="store_true", - help="Multi-surface per-task table (one column per file, labeled by surface)", + help="Multi-surface per-task table (one column per file, using its run label)", ) parser.add_argument( "--markdown", @@ -40,7 +40,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--no-dedupe", action="store_true", - help="Keep all rows (forensics); default is latest-wins per (task_id,rep,surface)", + help="Keep all rows (forensics); default is latest-wins per (task_id,rep,label)", ) arguments = parser.parse_args(argv) dedupe: DedupeMode = "none" if arguments.no_dedupe else "latest" @@ -64,7 +64,7 @@ def main(argv: list[str] | None = None) -> int: for path in paths: rows = load_rows(path, dedupe=dedupe) label = surface_label_for_file(path, rows) - # Disambiguate duplicate surface labels (e.g. two external files). + # Disambiguate duplicate run labels (e.g. two external files). label_root = label number = 2 while label in used_labels: diff --git a/evals/report/load.py b/evals/report/load.py index 0a7a4425..fbefa6ea 100644 --- a/evals/report/load.py +++ b/evals/report/load.py @@ -28,19 +28,19 @@ def is_infra_error_row(row: ResultRow) -> bool: """True when a row failed for infrastructure reasons, not task verification. Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, - ``infra_api``, ``infra_sdk``, …) is excluded from success-rate denominators. + ``infra_api``, …) is excluded from success-rate denominators. """ error_class = read_result(row).error_class return isinstance(error_class, str) and error_class.startswith("infra_") def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: - """Keep only the last row per (task_id, rep, surface); preserve key insertion order.""" + """Keep only the last row per (task_id, rep, label); preserve key insertion order.""" latest: dict[tuple[str, int, str], TaskResult] = {} order: list[tuple[str, int, str]] = [] for raw_row in rows: row = read_result(raw_row) - key = (row.task_id, row.rep, row.surface) + key = (row.task_id, row.rep, row.label) if key not in latest: order.append(key) latest[key] = row @@ -50,7 +50,7 @@ def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: """Load JSONL data rows (skip meta / missing task_id). - Default ``dedupe="latest"`` keeps the last row per (task_id, rep, surface) + Default ``dedupe="latest"`` keeps the last row per (task_id, rep, label) so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. """ rows: list[TaskResult] = [] @@ -78,10 +78,10 @@ def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: # Forensics: warn on duplicates but keep all. seen_keys: set[tuple[str, int, str]] = set() for row in rows: - key = (row.task_id, row.rep, row.surface) + key = (row.task_id, row.rep, row.label) if key in seen_keys: print( - f"warning: {path}: duplicate (task_id, rep, surface)={key} " + f"warning: {path}: duplicate (task_id, rep, label)={key} " f"(--no-dedupe keeps all rows; bare --out reuse double-counts)", file=sys.stderr, ) diff --git a/evals/report/table.py b/evals/report/table.py index a248e6c2..2f0237e4 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -137,7 +137,7 @@ def format_surface_cell(row: ResultRow | None) -> str: return "ERR" passed = "✅" if result.success else "❌" call_count = str(result.num_calls) - if result.classification == "external": + if result.server == "external": return f"{passed} {call_count}c" alternate = result.alternate_calls outside_set = result.out_of_set_calls @@ -180,7 +180,7 @@ def build_multi_surface_table( """Build a per-task × per-surface grid from labeled row sets. ``file_rows`` is a list of ``(column_label, rows)``. Column labels default - to each file's dominant ``surface`` field when the caller passes that label. + to each file's dominant ``label`` field when the caller passes that label. Rows are grouped by task and repetition. Single-rep columns retain the historical one-cell rendering; multi-rep columns aggregate all repetitions. """ @@ -241,7 +241,7 @@ def build_multi_surface_table( if row.success: successes += 1 calls += row.num_calls - if row.classification == "external": + if row.server == "external": mispicks_comparable = False else: alternate = row.alternate_calls @@ -348,12 +348,12 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) def surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: - """Pick a column label from the file's dominant surface field, else stem.""" + """Pick a column label from the file's dominant label field, else stem.""" counts: dict[str, int] = defaultdict(int) for row in rows: - surface = row.surface - if surface: - counts[str(surface)] += 1 + label = row.label + if label: + counts[str(label)] += 1 if counts: return max(counts, key=counts.get) # type: ignore[arg-type] return path.stem diff --git a/evals/results.py b/evals/results.py index cee442bb..8f493eb4 100644 --- a/evals/results.py +++ b/evals/results.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal RESULT_SCHEMA_VERSION = 1 @@ -55,10 +55,10 @@ class TaskResult: ts: str = "" git_sha: str = "" battery: str = "" - surface: str = "" + label: str = "" driver: str = "" provider: str | None = None - classification: str = "" + server: Literal["local", "external"] = "local" model: str | None = None requested_model: str | None = None requested_tier: str | None = None @@ -188,10 +188,10 @@ def usage_row(item: Usage) -> dict[str, int]: "ts": self.ts, "git_sha": self.git_sha, "battery": self.battery, - "surface": self.surface, + "label": self.label, "driver": self.driver, "provider": self.provider, - "classification": self.classification, + "server": self.server, "model": self.model, "requested_model": self.requested_model, "requested_tier": self.requested_tier, @@ -308,10 +308,10 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ts=str(row.get("ts") or ""), git_sha=str(row.get("git_sha") or ""), battery=str(row.get("battery") or ""), - surface=str(row.get("surface") or ""), + label=str(row.get("label") or ""), driver=str(row.get("driver") or ""), provider=(str(row["provider"]) if row.get("provider") is not None else None), - classification=str(row.get("classification") or ""), + server="external" if row.get("server") == "external" else "local", model=(str(row["model"]) if row.get("model") is not None else None), requested_model=(str(row["requested_model"]) if row.get("requested_model") is not None else None), requested_tier=(str(row["requested_tier"]) if row.get("requested_tier") is not None else None), diff --git a/evals/run.py b/evals/run.py deleted file mode 100644 index 4d5be019..00000000 --- a/evals/run.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Compatibility entry point for the Plane MCP eval harness. - -CLI concerns live in :mod:`evals.cli`; live execution and result bookkeeping -live in :mod:`evals.runner`. Existing imports and ``python -m evals.run`` remain -stable through this façade. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from evals import runner -from evals.cli import ( - API_MODEL_TIERS, - CLI_DRIVER_PROVIDERS, - CLI_MODEL_TIERS, - DEFAULT_OUT_DIR, - MODEL_TIERS, - cmd_dry_run, - cmd_list, - main, - parse_args, - resolve_model_for_driver, -) -from evals.runner import ( - KNOWN_SURFACES, - MAX_ITERATIONS, - MAX_TOKENS, - classify_call, - is_infra_cli_stop_reason, - is_meta_or_non_task_row, - load_resume_skip_keys, - make_run_meta_row, - maybe_write_run_meta, - run_agent_task_via_driver, - run_canary, - should_skip_resume_row, - stdio_server_env, -) - - -async def run_live( - tasks: list[dict[str, Any]], - *, - model_alias: str, - reps: int, - surface: str, - out_path: Path, - driver_name: str = "api", - provider: str = "anthropic", - server_cmd: list[str] | None = None, - server_env: dict[str, str] | None = None, - resume: bool = False, - record_result_payloads: bool = False, -) -> int: - """Delegate the legacy API while resolving provider-neutral model tiers.""" - driver_key = (driver_name or "api").strip().lower() - model_id = resolve_model_for_driver( - driver_key, - model_alias, - provider=provider if driver_key in ("api", "sdk") else None, - ) - return await runner.run_live( - tasks, - model_alias=model_alias, - reps=reps, - surface=surface, - out_path=out_path, - driver_name=driver_key, - provider=provider, - server_cmd=server_cmd, - server_env=server_env, - resume=resume, - record_result_payloads=record_result_payloads, - resolved_model_id=model_id, - ) - - -__all__ = [ - "API_MODEL_TIERS", - "CLI_DRIVER_PROVIDERS", - "CLI_MODEL_TIERS", - "DEFAULT_OUT_DIR", - "KNOWN_SURFACES", - "MAX_ITERATIONS", - "MAX_TOKENS", - "MODEL_TIERS", - "classify_call", - "cmd_dry_run", - "cmd_list", - "is_infra_cli_stop_reason", - "is_meta_or_non_task_row", - "load_resume_skip_keys", - "main", - "make_run_meta_row", - "maybe_write_run_meta", - "parse_args", - "resolve_model_for_driver", - "run_agent_task_via_driver", - "run_canary", - "run_live", - "should_skip_resume_row", - "stdio_server_env", -] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evals/runner/__init__.py b/evals/runner/__init__.py index bc6e8945..f33008aa 100644 --- a/evals/runner/__init__.py +++ b/evals/runner/__init__.py @@ -2,7 +2,6 @@ from .canary import run_canary from .live import ( - KNOWN_SURFACES, MAX_ITERATIONS, MAX_TOKENS, classify_call, @@ -15,7 +14,6 @@ from .resume import load_resume_skip_keys, should_skip_resume_row __all__ = [ - "KNOWN_SURFACES", "MAX_ITERATIONS", "MAX_TOKENS", "classify_call", diff --git a/evals/runner/canary.py b/evals/runner/canary.py index fa1f94fd..50f93faf 100644 --- a/evals/runner/canary.py +++ b/evals/runner/canary.py @@ -7,43 +7,29 @@ from typing import Any from evals.seed import make_plane_client, seed, teardown -from evals.tasks import TaskSkipped, battery_fingerprint, resolve_surface_tool_sets - -from .live import KNOWN_SURFACES +from evals.tasks import TaskSkipped, battery_fingerprint async def run_canary( tasks: list[dict[str, Any]], *, - surface: str, + label: str, ) -> int: """Seed + verify(empty agent) + teardown per task; no driver/model. Passes only when every verifier returns falsy ok on a do-nothing agent. Any ok=True is a broken verifier (false positive). """ - surface = (surface or "full").strip().lower() - if surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)}", - file=sys.stderr, - ) - return 2 - + label = (label or "local").strip() or "local" battery = battery_fingerprint(tasks) plane, _workspace_slug = make_plane_client() - print(f"canary battery={battery} surface={surface} tasks={[task['id'] for task in tasks]}") + print(f"canary battery={battery} label={label} tasks={[task['id'] for task in tasks]}") broken: list[str] = [] verified_count = 0 empty_agent = {"final_text": "", "calls": []} for task in tasks: - surface_sets = resolve_surface_tool_sets(task, surface) - if surface_sets.get("skip"): - print(f" {task['id']} SKIPPED (surface): {surface_sets['skip']}") - continue - context: dict[str, Any] = {} task_needs = set(task.get("needs") or set()) try: @@ -82,7 +68,8 @@ async def run_canary( return 1 if verified_count == 0: print( - "error: canary verified 0 tasks (all skipped by surface/plan gates) — nothing exercised; refusing exit 0", + "error: canary verified 0 tasks (all skipped by environment/fixture gates) " + "— nothing exercised; refusing exit 0", file=sys.stderr, ) return 1 diff --git a/evals/runner/live.py b/evals/runner/live.py index f73cf725..3c3668ba 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -20,18 +20,12 @@ TaskSkipped, battery_fingerprint, format_task_prompt, - resolve_surface_tool_sets, task_author, ) from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision from .resume import load_resume_skip_keys -# Surfaces the harness can run. ``full`` = legacy 177-tool stdio (default). -# ``v2`` / ``v2-schema`` set PLANE_MCP_SURFACE in the child env -# (see plane_mcp.v2.choose_stdio_mcp). -KNOWN_SURFACES = frozenset({"full", "v2", "v2-schema"}) - MAX_ITERATIONS = 15 MAX_TOKENS = 8192 @@ -53,14 +47,8 @@ def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: return "out_of_set" -def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = None) -> dict[str, str]: - """Build MCP stdio env from scratch — never inherit os.environ (F6). - - ``surface=v2`` sets ``PLANE_MCP_SURFACE=v2`` so the child process serves the - v2 tool registry. ``surface=full`` leaves the var unset (legacy default). - Other surface names (external servers under benchmark) set nothing; their - selection mechanism comes in via ``extra`` (--server-env) or --server-cmd args. - """ +def stdio_server_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6).""" environment: dict[str, str] = {} if path := os.environ.get("PATH"): environment["PATH"] = path @@ -69,10 +57,6 @@ def stdio_server_env(*, surface: str = "full", extra: dict[str, str] | None = No environment["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] environment["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] environment["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") - if surface == "v2": - environment["PLANE_MCP_SURFACE"] = "v2" - elif surface == "v2-schema": - environment["PLANE_MCP_SURFACE"] = "v2-schema" if extra: environment.update(extra) return environment @@ -112,7 +96,6 @@ async def run_agent_task_via_driver( task: dict[str, Any], ctx: dict[str, Any], workspace_slug: str, - surface: str = "full", optimal_tools: set[str] | None = None, alternate_tools: set[str] | None = None, server_env: dict[str, str] | None = None, @@ -125,7 +108,7 @@ async def run_agent_task_via_driver( alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" - mcp_env = stdio_server_env(surface=surface, extra=server_env) + mcp_env = stdio_server_env(extra=server_env) # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. agent_run = await asyncio.to_thread( driver.run_task, @@ -148,7 +131,7 @@ def _make_task_row( *, run_id: str, git_revision: str, - surface: str, + label: str, driver_name: str, provider: str | None, model_id: str | None, @@ -157,17 +140,17 @@ def _make_task_row( task: dict[str, Any], repetition: int, battery: str, - classification: str, + server: str, ) -> TaskResult: return TaskResult( run_id=run_id, ts=datetime.now(timezone.utc).isoformat(), git_sha=git_revision, battery=battery, - surface=surface, + label=label, driver=driver_name, provider=provider, - classification=classification, + server=server, model=model_id, requested_model=model_request, requested_tier=requested_tier, @@ -185,10 +168,9 @@ async def _run_task_repetition( workspace_slug: str, task: dict[str, Any], repetition: int, - surface_sets: dict[str, Any], run_id: str, git_revision: str, - surface: str, + label: str, driver_name: str, provider_id: str | None, model_id: str | None, @@ -204,7 +186,7 @@ async def _run_task_repetition( row = _make_task_row( run_id=run_id, git_revision=git_revision, - surface=surface, + label=label, driver_name=driver_name, provider=provider_id, model_id=model_id, @@ -213,150 +195,134 @@ async def _run_task_repetition( task=task, repetition=repetition, battery=battery, - classification=str(surface_sets["classification"]), + server="external" if external else "local", ) try: - # Surface-unsupported tasks: record skip, no seed/agent. - if surface_sets.get("skip"): - reason = surface_sets["skip"] - row.skipped = reason - row.verify_note = reason - print(f" {task['id']} rep={repetition} SKIPPED: {reason}") - else: - task_needs = set(task.get("needs") or set()) - # Seed wrap: TaskSkipped → skip; other failures → infra_seed. - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + if context.get("project_name"): print( - f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + f" orphaned project may remain: {context['project_name']}", file=sys.stderr, ) - if context.get("project_name"): + else: + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + row.skipped = reason + row.verify_note = reason + print(f" {task['id']} rep={repetition} SKIPPED: {reason}") + else: + agent: TaskResult | None = None + # Agent wrap: API failures and CLI failures are infrastructure. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + agent = await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=context, + workspace_slug=workspace_slug, + optimal_tools=set(task["optimal_tools"]), + alternate_tools=set(task["alternate_tools"]), + server_env=server_env, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" print( - f" orphaned project may remain: {context['project_name']}", + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", file=sys.stderr, ) - else: - if "bug_type" in task_needs and not context.get("bug_type"): - reason = context.get("bug_type_skip_reason") or "bug_type unavailable" - row.skipped = reason - row.verify_note = reason - print(f" {task['id']} rep={repetition} SKIPPED: {reason}") - else: - agent: TaskResult | None = None - # Agent wrap: API failures and CLI failures are infrastructure. - # Contained CLI stops (timeout / error subtypes) return AgentRun. - try: - agent = await run_agent_task_via_driver( - driver=driver, - model_id=model_id, - task=task, - ctx=context, - workspace_slug=workspace_slug, - surface=surface, - optimal_tools=surface_sets["optimal_tools"], - alternate_tools=surface_sets["alternate_tools"], - server_env=server_env, - ) - except PromptBindError as exc: - # Empty/missing seed IDs in the prompt — not an agent failure. + agent = None + except Exception as exc: + if is_api_driver: + agent_error_class = "infra_api" + else: + agent_error_class = "infra_cli" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = agent_error_class + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", + file=sys.stderr, + ) + agent = None + if agent is not None: + row.apply_agent_result(agent) + # Driver-level requested_model is the resolved ID. + # Restore run-level intent and retain both identities. + row.requested_model = model_alias + row.requested_tier = requested_tier + row.resolved_model = model_id + if external: + # Foreign tool names are not comparable to our catalog. + row.alternate_calls = None + row.out_of_set_calls = None + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.stop_reason + if driver_name.endswith("-cli") and is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - agent = None - except Exception as exc: - if driver_name == "sdk": - agent_error_class = "infra_sdk" - elif is_api_driver: - agent_error_class = "infra_api" + row.error_class = "infra_cli" + if stop_reason == "timeout": + row.error = _timeout_error_message(agent) else: - agent_error_class = "infra_cli" - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = agent_error_class + notes = [note for note in agent.driver_notes if isinstance(note, str)] + detail = "; ".join(notes) if notes else str(stop_reason) + row.error = detail row.verify_note = "" print( - f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", + f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", file=sys.stderr, ) - agent = None - - if agent is not None: - row.apply_agent_result(agent) - # Driver-level requested_model is the resolved ID. - # Restore run-level intent and retain both identities. - row.requested_model = model_alias - row.requested_tier = requested_tier - row.resolved_model = model_id - if external: - # Empty overlay sets would classify every call - # out-of-set; null the counters instead. - row.alternate_calls = None - row.out_of_set_calls = None - - # CLI infra stops: timeout + error subtypes except error_max_turns. - stop_reason = agent.stop_reason - if driver_name.endswith("-cli") and is_infra_cli_stop_reason( - str(stop_reason) if stop_reason is not None else None - ): + else: + verify = task["verify"] + try: + agent_row = agent.to_row() + ok, note = await verify( + plane, + context, + { + "final_text": agent.final_text, + "calls": agent_row["calls"], + }, + ) + row.success = bool(ok) + row.verify_note = note + print(f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}") + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + except Exception as exc: row.success = False - row.error_class = "infra_cli" - if stop_reason == "timeout": - row.error = _timeout_error_message(agent) - else: - notes = [note for note in agent.driver_notes if isinstance(note, str)] - detail = "; ".join(notes) if notes else str(stop_reason) - row.error = detail + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" row.verify_note = "" print( - f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", + f" {task['id']} rep={repetition} ERROR[task]: {exc}", file=sys.stderr, ) - else: - verify = task["verify"] - try: - agent_row = agent.to_row() - ok, note = await verify( - plane, - context, - { - "final_text": agent.final_text, - "calls": agent_row["calls"], - }, - ) - row.success = bool(ok) - row.verify_note = note - print( - f" {task['id']} rep={repetition} success={ok} " - f"calls={agent.num_calls} note={note!r}" - ) - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "task" - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[task]: {exc}", - file=sys.stderr, - ) except Exception as exc: # Anything outside seed/driver/verify wraps. row.success = False @@ -384,7 +350,7 @@ async def run_live( *, model_alias: str, reps: int, - surface: str, + label: str, out_path: Path, driver_name: str = "api", provider: str = "anthropic", @@ -394,15 +360,8 @@ async def run_live( record_result_payloads: bool = False, resolved_model_id: str | None = None, ) -> int: - surface = (surface or "full").strip().lower() + label = (label or "local").strip() or "local" external = server_cmd is not None - if not external and surface not in KNOWN_SURFACES: - print( - f"error: unknown --surface {surface!r}; expected one of {sorted(KNOWN_SURFACES)} " - "(or pass --server-cmd for an external surface)", - file=sys.stderr, - ) - return 2 driver_name = (driver_name or "api").strip().lower() if driver_name not in KNOWN_DRIVERS: @@ -412,7 +371,7 @@ async def run_live( ) return 2 provider = (provider or "anthropic").strip().lower() - is_api_driver = driver_name in ("api", "sdk") + is_api_driver = driver_name == "api" provider_id = provider if is_api_driver else None model_id = resolved_model_id if resolved_model_id is not None else model_alias requested_tier = model_alias if model_alias in MODEL_TIERS else None @@ -422,12 +381,12 @@ async def run_live( battery = battery_fingerprint(tasks) out_path.parent.mkdir(parents=True, exist_ok=True) - resume_skip: set[tuple[str, int]] = set() + resume_skip: set[tuple[str, int, str]] = set() if resume: try: resume_skip, skip_count, retry_count = load_resume_skip_keys( out_path, - surface=surface, + label=label, battery=battery, model=model_id, driver=driver_name, @@ -441,7 +400,8 @@ async def run_live( # First line of a new/empty file is a meta header (skipped by loaders). meta = make_run_meta_row( run_id=run_id, - surface=surface, + label=label, + server="external" if external else "local", battery=battery, model=model_id, requested_model=model_alias, @@ -452,7 +412,7 @@ async def run_live( git_sha=git_revision, ) if maybe_write_run_meta(out_path, meta): - print(f"wrote meta header battery={battery} surface={surface}") + print(f"wrote meta header battery={battery} label={label}") plane, workspace_slug = make_plane_client() # User chose --driver explicitly: codex live is allowed (they own the quota). @@ -472,25 +432,14 @@ async def run_live( print( f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} " f"requested_model={model_alias} resolved_model={model_id} " - f"surface={surface} tasks={[task['id'] for task in tasks]} reps={reps}" + f"label={label} tasks={[task['id'] for task in tasks]} reps={reps}" ) print(f"writing {out_path}") with out_path.open("a", encoding="utf-8") as file: for task in tasks: - if external: - # Foreign tool names have no overlay sets: no skips, no - # mispick classification — success/calls/errors only. - surface_sets = { - "skip": None, - "optimal_tools": set(), - "alternate_tools": set(), - "classification": "external", - } - else: - surface_sets = resolve_surface_tool_sets(task, surface) for repetition in range(reps): - if (task["id"], repetition) in resume_skip: + if (task["id"], repetition, label) in resume_skip: print(f" {task['id']} rep={repetition} RESUME_SKIP") continue row = await _run_task_repetition( @@ -499,10 +448,9 @@ async def run_live( workspace_slug=workspace_slug, task=task, repetition=repetition, - surface_sets=surface_sets, run_id=run_id, git_revision=git_revision, - surface=surface, + label=label, driver_name=driver_name, provider_id=provider_id, model_id=model_id, diff --git a/evals/runner/meta.py b/evals/runner/meta.py index 9b395bdd..231555f3 100644 --- a/evals/runner/meta.py +++ b/evals/runner/meta.py @@ -36,7 +36,8 @@ def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: def make_run_meta_row( *, run_id: str, - surface: str, + label: str, + server: str, battery: str, model: str | None, driver: str, @@ -52,7 +53,8 @@ def make_run_meta_row( "schema_version": RESULT_SCHEMA_VERSION, "row_type": "meta", "run_id": run_id, - "surface": surface, + "label": label, + "server": server, "battery": battery, "model": model, "requested_model": requested_model if requested_model is not None else model, diff --git a/evals/runner/resume.py b/evals/runner/resume.py index bee70ed1..197aee00 100644 --- a/evals/runner/resume.py +++ b/evals/runner/resume.py @@ -17,7 +17,7 @@ def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. Rows with ``skipped`` set are treated as complete and are not retried (intentional: - surface/plan skips are stable outcomes, not infra failures). + environment/fixture skips are stable outcomes, not infra failures). Pure function — unit-tested without the live battery. """ result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) @@ -41,8 +41,8 @@ def _resume_field_mismatch( raw = row.get(field) if raw is None or raw == "": return None # back-compat: older rows without the key pass - # Surface/driver/provider compare case-insensitively; battery/model are exact. - if field in ("surface", "driver", "provider"): + # Driver/provider compare case-insensitively; label/battery/model are exact. + if field in ("driver", "provider"): received, wanted = str(raw).strip().lower(), expected.strip().lower() else: received, wanted = str(raw).strip(), expected.strip() @@ -54,24 +54,24 @@ def _resume_field_mismatch( def load_resume_skip_keys( path: Path, *, - surface: str, + label: str, battery: str | None = None, model: str | None = None, driver: str | None = None, provider: str | None = None, -) -> tuple[set[tuple[str, int]], int, int]: - """Load existing JSONL rows and decide which (task_id, rep) pairs to skip. +) -> tuple[set[tuple[str, int, str]], int, int]: + """Load existing JSONL rows and decide which (task_id, rep, label) keys to skip. Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` - (keys that still need a re-run). Raises ``SystemExit`` when a row's surface / + (keys that still need a re-run). Raises ``SystemExit`` when a row's label / battery / model / driver / provider disagrees with the current run (missing keys pass for back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. """ if not path.is_file(): return set(), 0, 0 - skip_keys: set[tuple[str, int]] = set() - seen: set[tuple[str, int]] = set() + skip_keys: set[tuple[str, int, str]] = set() + seen: set[tuple[str, int, str]] = set() with path.open(encoding="utf-8") as file: for line_number, line in enumerate(file, start=1): line = line.strip() @@ -88,7 +88,7 @@ def load_resume_skip_keys( if not isinstance(row, dict): continue for field, expected in ( - ("surface", surface), + ("label", label), ("battery", battery), ("driver", driver), ("provider", provider), @@ -113,7 +113,7 @@ def load_resume_skip_keys( result = TaskResult.from_row(row) if not result.task_id: continue - key = (result.task_id, result.rep) + key = (result.task_id, result.rep, result.label) seen.add(key) if should_skip_resume_row(result): skip_keys.add(key) diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index 472e454d..c6c6fe02 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -128,71 +128,6 @@ TASKS_BY_ID: dict[str, dict[str, Any]] = {task["id"]: task for task in TASKS} -def resolve_surface_tool_sets( - task: dict[str, Any], - surface: str, -) -> dict[str, Any]: - """Resolve optimal/alternate tool sets for a surface. - - Returns a dict with: - - skip (str | None): if set, the runner should SKIP the task on this surface - - optimal_tools / alternate_tools: classification sets - - optimal_calls: optional override - - classification: ``exact`` when an overlay or full/legacy sets apply; - ``approximate`` when falling back to flat legacy-named sets on a non-full - surface that has no overlay - """ - surface = (surface or "full").strip().lower() - overlays = task.get("surface_tools") or {} - - if surface in ("full", "legacy", ""): - return { - "skip": None, - "optimal_tools": set(task["optimal_tools"]), - "alternate_tools": set(task["alternate_tools"]), - "optimal_calls": task.get("optimal_calls"), - "classification": "exact", - } - - ov = overlays.get(surface) - # v2-schema is a superset of v2 for *supported* tools, but schema adds none of - # the long-tail APIs (worklog summary, activities, release tags, customer - # property values). Inherit the full v2 overlay — including expected_skip / - # unsupported — when no schema-specific entry exists. - if ov is None and surface == "v2-schema": - ov = overlays.get("v2") - - if ov is None: - return { - "skip": None, - "optimal_tools": set(task["optimal_tools"]), - "alternate_tools": set(task["alternate_tools"]), - "optimal_calls": task.get("optimal_calls"), - "classification": "approximate", - } - - if ov.get("unsupported") or ov.get("expected_skip"): - return { - "skip": ov.get("reason") or f"task {task.get('id')} unsupported on surface {surface}", - "optimal_tools": set(), - "alternate_tools": set(), - "optimal_calls": None, - "classification": "exact", - } - - optimal = set(ov["optimal_tools"]) - alternate = set(ov["alternate_tools"]) - if not optimal.isdisjoint(alternate): - raise ValueError(f"{task.get('id')}/{surface}: optimal/alternate overlap") - return { - "skip": None, - "optimal_tools": optimal, - "alternate_tools": alternate, - "optimal_calls": ov.get("optimal_calls", task.get("optimal_calls")), - "classification": "exact", - } - - def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: """Return tasks filtered by id list (None = all).""" if ids is None: @@ -208,35 +143,11 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -def _serialize_surface_tools(surface_tools: dict[str, Any] | None) -> dict[str, Any]: - """Stable JSON-friendly form of a task's surface_tools overlay.""" - if not surface_tools: - return {} - out: dict[str, Any] = {} - for surface in sorted(surface_tools): - ov = surface_tools[surface] or {} - if not isinstance(ov, dict): - out[surface] = ov - continue - entry: dict[str, Any] = {} - for key in sorted(ov): - val = ov[key] - if isinstance(val, set | frozenset): - entry[key] = sorted(val) - elif isinstance(val, list | tuple): - entry[key] = list(val) - else: - entry[key] = val - out[surface] = entry - return out - - def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: """Stable short hash of the task battery used for a run. SHA-256 (first 12 hex chars) over a canonical serialization of every task - sorted by id: id, prompt, sorted optimal/alternate tools, optimal_calls, - and the surface_tools overlay (sets sorted, keys sorted). + sorted by id: id, prompt, sorted optimal/alternate tools, and optimal_calls. Ceilings (intentionally *not* covered by the hash): - Verifier functions and ``needs`` fixtures do not alter the fingerprint — @@ -254,7 +165,6 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "optimal_tools": sorted(t.get("optimal_tools") or []), "alternate_tools": sorted(t.get("alternate_tools") or []), "optimal_calls": t.get("optimal_calls"), - "surface_tools": _serialize_surface_tools(t.get("surface_tools")), } ) blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) @@ -282,7 +192,6 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "reports_contract_value", "reports_contract_values", "reports_exact_int", - "resolve_surface_tool_sets", "state_group", "state_name", "task_author", diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py index b64699ca..df56b1ef 100644 --- a/evals/tasks/cross.py +++ b/evals/tasks/cross.py @@ -115,23 +115,6 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "search_work_items", "list_projects", }, - "surface_tools": { - "v2": { - "optimal_calls": 4, - "optimal_tools": { - "list_customers", - "create_customer", - "log_customer_request", - "link_customer_work_items", - }, - "alternate_tools": { - "get_customer", - "find_work_items", - "update_customer", - "search_projects", - }, - }, - }, # No pre-seeded customer — agent creates; items needed for link target. "needs": {"items"}, "verify": verify_c1, @@ -184,17 +167,6 @@ async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_release_work_items", "update_release_changelog", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"get_release"}, - "alternate_tools": { - "list_releases", - "assign_to_release", - "get_workspace_context", - }, - }, - }, "needs": {"release"}, "verify": verify_c2, } diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 93c33519..41fae19f 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -113,13 +113,6 @@ async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "search_work_items", "retrieve_work_item_by_identifier", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "find_work_items", "search_projects"}, - }, - }, "needs": {"items"}, "verify": verify_i1, } @@ -159,20 +152,6 @@ async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "search_work_items", "list_states", }, - "surface_tools": { - "v2": { - # get_work_item requires UUIDs (forwards work_item_id directly). - # PROJ-N on v2 is resolved via find_work_items (list/filter). - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "list_states", - "search_projects", - "get_workspace_context", - }, - }, - }, "needs": {"items"}, "verify": verify_i2, } @@ -214,13 +193,6 @@ async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_items", "retrieve_cycle", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"assign_to_cycle"}, - "alternate_tools": {"list_cycles", "find_work_items", "get_work_item"}, - }, - }, "needs": {"items", "cycles"}, "verify": verify_i3, } @@ -260,14 +232,6 @@ async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_work_item", "list_work_items", }, - "surface_tools": { - "v2": { - # Default v2 update_work_item accepts labels; no manage_work_item_label. - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "list_labels", "find_work_items"}, - }, - }, "needs": {"items", "labels"}, "verify": verify_i4, } @@ -300,13 +264,6 @@ async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_items", "search_work_items", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": {"get_work_item", "find_work_items"}, - }, - }, "needs": {"items"}, "verify": verify_i5, } @@ -374,12 +331,6 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_work_item", "list_projects", }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ("L1 needs get_project_worklog_summary (legacy project tool) — not on the default v2 surface"), - }, - }, "needs": {"items"}, "verify": verify_l1, } @@ -424,15 +375,6 @@ async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_item_comments", "retrieve_work_item_activity", }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ( - "L2 needs list_work_item_activities — not on the default v2 surface " - "(v2 has comments include= but not the activities feed)" - ), - }, - }, "needs": {"items", "activity_feed"}, "verify": verify_l2, } @@ -474,12 +416,6 @@ async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_releases", "update_release_tag", }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": "L3 needs create_release_tag — not on the default v2 surface", - }, - }, "needs": set(), # workspace-level tag; no project fixture required "verify": verify_l3, } @@ -566,14 +502,6 @@ async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_customer", "update_customer_property", }, - "surface_tools": { - "v2": { - "expected_skip": True, - "reason": ( - "L4 needs create_customer_property / set_customer_property_values — not on the default v2 surface" - ), - }, - }, "needs": {"customer"}, "verify": verify_l4, } @@ -614,18 +542,6 @@ async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_work_item", "get_work_item_attachment_download_url", }, - "surface_tools": { - "v2": { - # Achievable on default v2 via include=attachments on get_work_item. - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "get_work_item"}, - "alternate_tools": { - "search_projects", - "get_workspace_context", - "list_states", - }, - }, - }, "needs": {"items"}, "verify": verify_l5, } diff --git a/evals/tasks/read.py b/evals/tasks/read.py index cb12ab1b..c0f7f0b8 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -57,19 +57,6 @@ async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_projects", "list_states", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "list_states", - "get_workspace_context", - "get_pql_reference", - }, - }, - }, "needs": {"items"}, "verify": verify_r1, } @@ -102,20 +89,6 @@ async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_states", "get_pql_reference", }, - "surface_tools": { - "v2": { - # No count tool on v2 — find_work_items with priority/state filters is optimal. - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "list_states", - "get_workspace_context", - "get_pql_reference", - }, - }, - }, "needs": {"items"}, "verify": verify_r2, } @@ -149,18 +122,6 @@ async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "get_pql_reference", "retrieve_work_item", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "get_workspace_context", - "get_work_item", - "search_projects", - "get_pql_reference", - }, - }, - }, "needs": {"items"}, "verify": verify_r3, } @@ -217,19 +178,6 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_projects", "get_pql_reference", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"find_work_items"}, - "alternate_tools": { - "list_cycles", - "get_work_item", - "get_pql_reference", - "search_projects", - "get_workspace_context", - }, - }, - }, "needs": {"items", "cycles"}, "verify": verify_r4, } @@ -262,19 +210,6 @@ async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_item_activities", "list_projects", }, - "surface_tools": { - "v2": { - # include= depth: single get_work_item with include=comments after resolve, - # or find + get with include. - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "get_work_item"}, - "alternate_tools": { - "search_projects", - "get_workspace_context", - "create_comment", - }, - }, - }, "needs": {"items"}, "verify": verify_r5, } @@ -308,28 +243,6 @@ async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_project", "get_pql_reference", }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"search_projects", "find_work_items", "get_workspace_context"}, - "alternate_tools": { - "get_work_item", - "get_pql_reference", - "list_states", - }, - }, - # Type id resolution cleaner on v2-schema - "v2-schema": { - "optimal_calls": 3, - "optimal_tools": {"search_projects", "find_work_items", "resolve_work_item_type"}, - "alternate_tools": { - "list_work_item_types", - "get_workspace_context", - "get_work_item", - "get_pql_reference", - }, - }, - }, "needs": {"items", "bug_type", "second_project"}, "verify": verify_r6, } @@ -375,18 +288,6 @@ async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "search_work_items", "list_projects", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"list_available_transitions"}, - "alternate_tools": { - "find_work_items", - "get_work_item", - "list_states", - "search_projects", - }, - }, - }, "needs": {"items"}, "verify": verify_r7, } diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py index e740d02a..8984bff2 100644 --- a/evals/tasks/schema.py +++ b/evals/tasks/schema.py @@ -106,33 +106,6 @@ async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "import_work_item_types_to_project", "update_project_features", }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S1 needs work-item-type/property schema tools " - "(resolve_work_item_type, create_work_item_property) which are " - "not on the default v2 surface — use --surface v2-schema" - ), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": { - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "list_work_item_properties", - "add_property_option", - "search_projects", - "get_workspace_context", - "get_features", - "configure_features", - "update_work_item_type", - }, - }, - }, "needs": {"bug_type"}, "verify": verify_s1, } @@ -203,30 +176,6 @@ async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "search_work_items", "update_project_estimate", }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S2 needs configure_estimate (schema tier) to create the Fibonacci " - "scale — default v2 has no estimate schema tools. " - "(v2 update_work_item does accept estimate_point.) Use v2-schema." - ), - }, - "v2-schema": { - # configure_estimate creates scale+points+link in one call; - # update_work_item(estimate_point="5") resolves the value server-side. - "optimal_calls": 2, - "optimal_tools": {"configure_estimate", "update_work_item"}, - "alternate_tools": { - "search_projects", - "get_features", - "find_work_items", - "get_work_item", - "get_workspace_context", - "bulk_update_work_items", - }, - }, - }, "needs": {"items"}, "verify": verify_s2, } @@ -311,29 +260,6 @@ async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "manage_work_item_type_properties", "update_project_features", }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S3 needs resolve_work_item_type + create_work_item_property (schema tier) — use --surface v2-schema" - ), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": { - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "list_work_item_properties", - "update_work_item_type", - "search_projects", - "get_features", - "configure_features", - }, - }, - }, "needs": set(), "verify": verify_s3, } @@ -404,18 +330,6 @@ def _status_of(issue_id: str | None, title: str) -> int | None: "list_projects", "create_intake_work_item", }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"list_intake", "triage_intake"}, - "alternate_tools": { - "find_work_items", - "get_work_item", - "search_projects", - "get_workspace_context", - }, - }, - }, "needs": {"intake"}, "verify": verify_s4, } @@ -507,27 +421,6 @@ async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_project", "get_features", }, - "surface_tools": { - "v2": { - "unsupported": True, - "reason": ( - "S5 needs configure_features (schema tier) for project cycles/worklogs " - "and workspace customers — use --surface v2-schema" - ), - }, - "v2-schema": { - # 2 calls: configure_features(project, cycles+worklogs) + - # configure_features(customers=True) without project. - "optimal_calls": 2, - "optimal_tools": {"configure_features"}, - "alternate_tools": { - "get_features", - "search_projects", - "get_workspace_context", - "update_work_item", - }, - }, - }, # Seed leaves project cycles+worklogs and workspace customers off. "needs": {"leave_cycles_worklogs_off"}, "verify": verify_s5, diff --git a/evals/tasks/write.py b/evals/tasks/write.py index 00c676ba..b988d986 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -101,19 +101,6 @@ async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "update_work_item", "list_work_items", }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"get_workspace_context", "create_work_item"}, - "alternate_tools": { - "search_projects", - "list_labels", - "find_work_items", - "get_work_item", - "update_work_item", - }, - }, - }, "needs": {"labels"}, "verify": verify_w1, } @@ -146,18 +133,6 @@ async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_state", "list_projects", }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "update_work_item"}, - "alternate_tools": { - "list_states", - "get_work_item", - "list_available_transitions", - "search_projects", - }, - }, - }, "needs": {"items"}, "verify": verify_w2, } @@ -205,17 +180,6 @@ async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_item_comments", "list_projects", }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "create_comment"}, - "alternate_tools": { - "get_work_item", - "modify_comment", - "search_projects", - }, - }, - }, "needs": {"items"}, "verify": verify_w3, } @@ -264,23 +228,6 @@ async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "delete_label", "list_projects", }, - "surface_tools": { - "v2": { - # Default v2 has list_labels but no update_label (schema tier). - "unsupported": True, - "reason": ("W4 needs update_label which is only on the v2-schema surface — use --surface v2-schema"), - }, - "v2-schema": { - "optimal_calls": 2, - "optimal_tools": {"list_labels", "update_label"}, - "alternate_tools": { - "create_label", - "delete_label", - "search_projects", - "get_workspace_context", - }, - }, - }, "needs": {"labels"}, "verify": verify_w4, } @@ -355,18 +302,6 @@ async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_projects", "list_states", }, - "surface_tools": { - "v2": { - "optimal_calls": 5, - "optimal_tools": {"list_modules", "find_work_items", "archive_work_item"}, - "alternate_tools": { - "get_work_item", - "assign_to_module", - "search_projects", - "list_states", - }, - }, - }, "needs": {"module"}, "verify": verify_w5, } @@ -462,19 +397,6 @@ async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_items", "list_projects", }, - "surface_tools": { - "v2": { - # close_cycle with transfer_to is the consolidated path. - "optimal_calls": 2, - "optimal_tools": {"list_cycles", "close_cycle"}, - "alternate_tools": { - "assign_to_cycle", - "find_work_items", - "search_projects", - "get_workspace_context", - }, - }, - }, # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — # Plane rejects every edit to an ended cycle. See _seed_cycles. "needs": {"items", "cycles", "cycles_open_past"}, @@ -566,17 +488,6 @@ async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_work_item_links", "retrieve_work_item", }, - "surface_tools": { - "v2": { - "optimal_calls": 3, - "optimal_tools": {"find_work_items", "link_work_items", "add_work_item_link"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "update_work_item", - }, - }, - }, "needs": {"items"}, "verify": verify_w7, } @@ -617,17 +528,6 @@ async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "retrieve_work_item", "list_projects", }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "log_work"}, - "alternate_tools": { - "get_work_item", - "search_projects", - "update_work_item", - }, - }, - }, "needs": {"items"}, "verify": verify_w8, } @@ -679,17 +579,6 @@ async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "list_projects", "retrieve_work_item", }, - "surface_tools": { - "v2": { - "optimal_calls": 2, - "optimal_tools": {"find_work_items", "bulk_update_work_items"}, - "alternate_tools": { - "update_work_item", - "get_work_item", - "search_projects", - }, - }, - }, "needs": {"items"}, "verify": verify_w9, } @@ -725,18 +614,6 @@ async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu "retrieve_page", "attach_page_to_work_item", }, - "surface_tools": { - "v2": { - "optimal_calls": 1, - "optimal_tools": {"create_page"}, - "alternate_tools": { - "list_pages", - "get_page", - "search_projects", - "get_workspace_context", - }, - }, - }, "needs": set(), "verify": verify_w10, } diff --git a/tests/fixtures/evals_historical_rows.jsonl b/tests/fixtures/evals_historical_rows.jsonl index 51a88478..eee1306a 100644 --- a/tests/fixtures/evals_historical_rows.jsonl +++ b/tests/fixtures/evals_historical_rows.jsonl @@ -1,2 +1,2 @@ -{"run_id": "7a637f5a54664d3eb2fff9ac5a53fb43", "ts": "2026-08-12T17:59:05.498671+00:00", "git_sha": "5da71142cab2d9fd7e8f95be8192ccb17ac3d826", "battery": "6647676edc9e", "surface": "manish-v2", "driver": "codex-cli", "classification": "external", "model": "gpt-5.6-sol", "task_id": "L3", "author": "post-hoc-debias", "rep": 0, "success": true, "verify_note": "release tag 'eval-rc1' present", "skipped": null, "error": null, "error_class": null, "stop_reason": "end_turn", "hit_max_iterations": false, "calls": [{"tool": "release_tag", "class": "out_of_set", "args_chars": 43, "result_tokens": null, "result_chars": 1016, "result_kind": "text", "is_error": false, "duration_ms": 91, "action": "create", "result_tokens_skipped": "no API key / CLI driver has no count_tokens"}], "num_calls": 1, "errored_calls": 0, "alternate_calls": null, "out_of_set_calls": null, "total_result_tokens": 0, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 25.381, "client_tool_calls": [{"tool": "release_tag", "args_chars": 43, "raw_tool": "release_tag"}], "client_tool_call_count": 1, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_pair_mismatch": false, "token_count_failures": 0, "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff720-bf7b-75d2-9b23-eb0b635be673", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"], "result_tokens_skipped_reason": "CLI driver: count_tokens requires Anthropic API key; skipped", "usage": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 263426, "source": "codex_token_count"}} -{"run_id": "625464c995c646429f7cfbcb1a9f5166", "ts": "2026-08-13T03:37:23.554029+00:00", "git_sha": "adf653458ed5788e58acc5e2e9751143df942a5d", "battery": "6425dcc64404", "surface": "full", "driver": "codex-cli", "provider": null, "classification": "exact", "model": "gpt-5.6-sol", "requested_model": "gpt-5.6-sol", "task_id": "R2", "author": "claude", "rep": 0, "success": true, "verify_note": "final text names count 4", "skipped": null, "error": null, "error_class": null, "final_text": "I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4", "stop_reason": "end_turn", "hit_max_iterations": false, "result_pair_mismatch": false, "token_count_failures": 0, "result_tokens_estimated": true, "calls": [{"tool": "list_projects", "class": "alternate", "args_chars": 18, "result_tokens": 315, "result_chars": 1258, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 159}, {"tool": "count_work_items", "class": "optimal", "args_chars": 118, "result_tokens": 64, "result_chars": 253, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 107}], "num_calls": 2, "errored_calls": 0, "alternate_calls": 1, "out_of_set_calls": 0, "total_result_tokens": 379, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 29.679, "client_tool_calls": [{"tool": "list_projects", "args_chars": 18, "raw_tool": "list_projects"}, {"tool": "count_work_items", "args_chars": 118, "raw_tool": "count_work_items"}], "client_tool_call_count": 2, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_tokens_mode": "estimated", "result_token_count_method": "chars_div_4", "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff932-5598-7d62-9ab9-30c6bf5fca15", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"], "usage": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 292690, "source": "codex_token_count"}} +{"run_id": "7a637f5a54664d3eb2fff9ac5a53fb43", "ts": "2026-08-12T17:59:05.498671+00:00", "git_sha": "5da71142cab2d9fd7e8f95be8192ccb17ac3d826", "battery": "6647676edc9e", "label": "manish-v2", "driver": "codex-cli", "server": "external", "model": "gpt-5.6-sol", "task_id": "L3", "author": "post-hoc-debias", "rep": 0, "success": true, "verify_note": "release tag 'eval-rc1' present", "skipped": null, "error": null, "error_class": null, "stop_reason": "end_turn", "hit_max_iterations": false, "calls": [{"tool": "release_tag", "class": "out_of_set", "args_chars": 43, "result_tokens": null, "result_chars": 1016, "result_kind": "text", "is_error": false, "duration_ms": 91, "action": "create", "result_tokens_skipped": "no API key / CLI driver has no count_tokens"}], "num_calls": 1, "errored_calls": 0, "alternate_calls": null, "out_of_set_calls": null, "total_result_tokens": 0, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 25.381, "client_tool_calls": [{"tool": "release_tag", "args_chars": 43, "raw_tool": "release_tag"}], "client_tool_call_count": 1, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_pair_mismatch": false, "token_count_failures": 0, "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff720-bf7b-75d2-9b23-eb0b635be673", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"], "result_tokens_skipped_reason": "CLI driver: count_tokens requires Anthropic API key; skipped", "usage": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 263426, "source": "codex_token_count"}} +{"run_id": "625464c995c646429f7cfbcb1a9f5166", "ts": "2026-08-13T03:37:23.554029+00:00", "git_sha": "adf653458ed5788e58acc5e2e9751143df942a5d", "battery": "6425dcc64404", "label": "full", "driver": "codex-cli", "provider": null, "server": "local", "model": "gpt-5.6-sol", "requested_model": "gpt-5.6-sol", "task_id": "R2", "author": "claude", "rep": 0, "success": true, "verify_note": "final text names count 4", "skipped": null, "error": null, "error_class": null, "final_text": "I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4", "stop_reason": "end_turn", "hit_max_iterations": false, "result_pair_mismatch": false, "token_count_failures": 0, "result_tokens_estimated": true, "calls": [{"tool": "list_projects", "class": "alternate", "args_chars": 18, "result_tokens": 315, "result_chars": 1258, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 159}, {"tool": "count_work_items", "class": "optimal", "args_chars": 118, "result_tokens": 64, "result_chars": 253, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 107}], "num_calls": 2, "errored_calls": 0, "alternate_calls": 1, "out_of_set_calls": 0, "total_result_tokens": 379, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 29.679, "client_tool_calls": [{"tool": "list_projects", "args_chars": 18, "raw_tool": "list_projects"}, {"tool": "count_work_items", "args_chars": 118, "raw_tool": "count_work_items"}], "client_tool_call_count": 2, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_tokens_mode": "estimated", "result_token_count_method": "chars_div_4", "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff932-5598-7d62-9ab9-30c6bf5fca15", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"], "usage": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 292690, "source": "codex_token_count"}} diff --git a/tests/test_evals_catalog.py b/tests/test_evals_catalog.py index 23faf2ee..a58be195 100644 --- a/tests/test_evals_catalog.py +++ b/tests/test_evals_catalog.py @@ -8,11 +8,11 @@ from evals import seed as seed_mod from evals import tasks as tasks_mod -from evals.run import cmd_dry_run, cmd_list, parse_args +from evals.cli import cmd_dry_run, cmd_list, parse_args from evals.seed import seed_plan -from evals.tasks import TASKS, TASKS_BY_ID, get_tasks, resolve_surface_tool_sets +from evals.tasks import TASKS, TASKS_BY_ID, get_tasks -# DESIGN.md catalog ids (stable) + extras added for uncovered v2 families. +# DESIGN.md catalog ids (stable) + extras added for uncovered tool families. DESIGN_IDS = { "R1", "R2", @@ -123,96 +123,14 @@ def test_task_schema_invariants(): assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] assert callable(t["verify"]) assert isinstance(t.get("needs"), set) - # Overlays: optimal/alternate disjoint when present (skip capability-gap marks) - for surface, ov in (t.get("surface_tools") or {}).items(): - if ov.get("unsupported") or ov.get("expected_skip"): - continue - opt = set(ov["optimal_tools"]) - alt = set(ov["alternate_tools"]) - assert opt.isdisjoint(alt), f"{t['id']}/{surface} overlap" -def test_debias_tasks_author_and_v2_skips(): +def test_debias_tasks_author(): from evals.tasks import task_author for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: t = TASKS_BY_ID[tid] assert task_author(t) == "post-hoc-debias" - # L1–L4 are capability gaps on default v2 (+ inherited by v2-schema). - for tid in ("L1", "L2", "L3", "L4"): - for surface in ("v2", "v2-schema"): - skip = resolve_surface_tool_sets(TASKS_BY_ID[tid], surface)["skip"] - assert skip, f"{tid} should expected_skip on {surface}" - assert resolve_surface_tool_sets(TASKS_BY_ID[tid], "full")["skip"] is None - # L5 is achievable on v2 via get_work_item(include=attachments). - assert resolve_surface_tool_sets(TASKS_BY_ID["L5"], "v2")["skip"] is None - l5 = resolve_surface_tool_sets(TASKS_BY_ID["L5"], "v2") - assert l5["optimal_tools"] == {"find_work_items", "get_work_item"} - assert l5["optimal_calls"] == 2 - # I-class is runnable on v2 (raw call efficiency, not a capability gap) - for tid in ID_IN_HAND_IDS: - assert resolve_surface_tool_sets(TASKS_BY_ID[tid], "v2")["skip"] is None - # I2: PROJ-N is find_work_items, not get_work_item (UUID-only). - i2 = resolve_surface_tool_sets(TASKS_BY_ID["I2"], "v2") - assert i2["optimal_tools"] == {"find_work_items"} - assert "get_work_item" not in i2["optimal_tools"] - - -def test_v2_schema_inherits_expected_skip(): - """v2-schema must inherit L1 expected_skip (schema adds none of those APIs).""" - l1_v2 = resolve_surface_tool_sets(TASKS_BY_ID["L1"], "v2") - l1_schema = resolve_surface_tool_sets(TASKS_BY_ID["L1"], "v2-schema") - assert l1_v2["skip"] - assert l1_schema["skip"] == l1_v2["skip"] - # S1 remains schema-supported (explicit v2-schema overlay). - assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2")["skip"] - assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema")["skip"] is None - - -def test_every_task_resolves_on_full_v2_v2schema(): - for t in TASKS: - for surface in ("full", "v2", "v2-schema"): - out = resolve_surface_tool_sets(t, surface) - assert "classification" in out - assert out["classification"] in {"exact", "approximate"} - # skip is either None or a non-empty reason string - if out["skip"] is not None: - assert isinstance(out["skip"], str) and out["skip"] - - -def test_s1_w4_s2_s3_surface_skips(): - assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2")["skip"] - assert resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema")["skip"] is None - assert resolve_surface_tool_sets(TASKS_BY_ID["W4"], "v2")["skip"] - assert resolve_surface_tool_sets(TASKS_BY_ID["W4"], "v2-schema")["skip"] is None - assert resolve_surface_tool_sets(TASKS_BY_ID["S2"], "v2")["skip"] - # S2 became supported on v2-schema once update_work_item gained estimate_point. - s2 = resolve_surface_tool_sets(TASKS_BY_ID["S2"], "v2-schema") - assert s2["skip"] is None - assert s2["optimal_tools"] == {"configure_estimate", "update_work_item"} - assert s2["optimal_calls"] == 2 - assert resolve_surface_tool_sets(TASKS_BY_ID["S3"], "v2")["skip"] - assert resolve_surface_tool_sets(TASKS_BY_ID["S3"], "v2-schema")["skip"] is None - assert resolve_surface_tool_sets(TASKS_BY_ID["S5"], "v2")["skip"] - s5 = resolve_surface_tool_sets(TASKS_BY_ID["S5"], "v2-schema") - assert s5["skip"] is None - assert s5["optimal_tools"] == {"configure_features"} - assert s5["optimal_calls"] == 2 - full_s5 = resolve_surface_tool_sets(TASKS_BY_ID["S5"], "full") - assert full_s5["optimal_tools"] == {"update_project", "update_workspace_features"} - assert full_s5["optimal_calls"] == 2 - - -def test_v2_schema_inherits_v2_overlay_when_absent(): - """v2-schema with no own overlay uses the v2 sets (superset surface).""" - r1 = resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2-schema") - assert r1["skip"] is None - assert r1["classification"] == "exact" - assert r1["optimal_tools"] == {"find_work_items"} - # S1 has an explicit v2-schema overlay (not the unsupported v2 one). - s1 = resolve_surface_tool_sets(TASKS_BY_ID["S1"], "v2-schema") - assert s1["skip"] is None - assert "create_work_item_property" in s1["optimal_tools"] def test_w6_seeds_an_open_cycle(): @@ -225,25 +143,6 @@ def test_w6_seeds_an_open_cycle(): assert "cycles" in TASKS_BY_ID["W6"]["needs"] -def test_v2_overlays_use_v2_tool_names(): - """Spot-check headline v2 paths.""" - assert resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2")["optimal_tools"] == {"find_work_items"} - w6 = resolve_surface_tool_sets(TASKS_BY_ID["W6"], "v2") - assert "close_cycle" in w6["optimal_tools"] - s4 = resolve_surface_tool_sets(TASKS_BY_ID["S4"], "v2") - assert "triage_intake" in s4["optimal_tools"] and "list_intake" in s4["optimal_tools"] - w9 = resolve_surface_tool_sets(TASKS_BY_ID["W9"], "v2") - assert "bulk_update_work_items" in w9["optimal_tools"] - w10 = resolve_surface_tool_sets(TASKS_BY_ID["W10"], "v2") - assert "create_page" in w10["optimal_tools"] - r7 = resolve_surface_tool_sets(TASKS_BY_ID["R7"], "v2") - assert "list_available_transitions" in r7["optimal_tools"] - c2 = resolve_surface_tool_sets(TASKS_BY_ID["C2"], "v2") - assert "get_release" in c2["optimal_tools"] - c1 = resolve_surface_tool_sets(TASKS_BY_ID["C1"], "v2") - assert "create_customer" in c1["optimal_tools"] - - def test_seed_plan_covers_all_groups(): groups = { "items", @@ -311,8 +210,10 @@ def test_cmd_dry_run_all_tasks(capsys): def test_parse_args_list(): - a = parse_args(["--list"]) + a = parse_args(["--list", "--label", "candidate-build"]) assert a.list is True + assert a.label == "candidate-build" + assert parse_args(["--list"]).label == "local" def test_tasks_module_has_no_hardcoded_uuids(): diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index 394d0657..c4db0492 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -13,6 +13,7 @@ import pytest +from evals.cli import parse_args from evals.drivers import ( KNOWN_DRIVERS, AgentRun, @@ -33,7 +34,7 @@ write_claude_mcp_config, ) from evals.drivers.token_counting import estimate_result_tokens -from evals.run import classify_call, parse_args, stdio_server_env +from evals.runner.live import classify_call, stdio_server_env # --------------------------------------------------------------------------- # Fixtures (constructed — never captured from live CLIs) @@ -539,7 +540,7 @@ def fake_run(cmd, **kwargs): "PLANE_API_KEY": "key", "PLANE_WORKSPACE_SLUG": "slug", "PLANE_BASE_URL": "https://api.example", - "PLANE_MCP_SURFACE": "v2", + "CUSTOM_SETTING": "enabled", "PATH": "/usr/bin", }, model="sonnet", @@ -586,11 +587,11 @@ def fake_run(cmd, **kwargs): driver = ClaudeCliDriver( runner=fake_run, - server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--v2"], + server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], ) driver.run_task( "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_MCP_TOOLS_VERSION": "v2"}, + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, model="sonnet", max_turns=3, cwd=tmp_path, @@ -600,9 +601,14 @@ def fake_run(cmd, **kwargs): assert server["args"][:3] == ["-m", "evals.proxy", "--log"] assert "--" in server["args"] dash = server["args"].index("--") - assert server["args"][dash + 1 :] == ["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--v2"] - # PLANE_-prefixed env (incl. foreign selection vars) passes through to the child. - assert server["env"]["PLANE_MCP_TOOLS_VERSION"] == "v2" + assert server["args"][dash + 1 :] == [ + "/elsewhere/.venv/bin/plane-mcp-server", + "stdio", + "--mode", + "candidate", + ] + # Explicit foreign selection variables pass through to the child. + assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" def test_agent_run_dict_keeps_action_arg(): @@ -846,12 +852,11 @@ def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monk def test_known_drivers(): - assert KNOWN_DRIVERS == {"api", "sdk", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} -def test_get_driver_api_and_sdk_alias(): +def test_get_driver_api(): assert isinstance(get_driver("api"), ApiDriver) - assert isinstance(get_driver("sdk"), ApiDriver) assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) assert isinstance(get_driver("codex-cli"), CodexCliDriver) @@ -872,8 +877,7 @@ def test_stdio_env_still_works_for_cli_drivers(monkeypatch): monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - env = stdio_server_env(surface="v2") - assert env["PLANE_MCP_SURFACE"] == "v2" + env = stdio_server_env() assert env["PLANE_API_KEY"] == "k" assert "ANTHROPIC_API_KEY" not in env diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 2e730d0e..8b3ae694 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -12,31 +12,31 @@ import pytest from plane.errors.errors import HttpError +from evals import cli as run_mod from evals import report as report_mod -from evals import run as run_mod from evals import seed as seed_mod from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize from evals.results import RESULT_SCHEMA_VERSION, TaskResult -from evals.run import ( +from evals.runner import canary as runner_canary +from evals.runner import ( is_infra_cli_stop_reason, load_resume_skip_keys, run_canary, run_live, should_skip_resume_row, ) -from evals.runner import canary as runner_canary from evals.runner import live as runner_live from evals.seed import create_project_with_identifier_retry, is_identifier_collision -from evals.tasks import battery_fingerprint, task_author +from evals.tasks import TaskSkipped, battery_fingerprint, task_author # Pinned hash of the fixed synthetic catalog in test_battery_fingerprint_stable_and_sensitive. # Recompute only if the serialization format of battery_fingerprint changes deliberately. -PINNED_SYNTHETIC_BATTERY = "81be78bde8c7" +PINNED_SYNTHETIC_BATTERY = "eea5abf36382" def _data_rows(path: Path) -> list[dict]: - """Parse JSONL skipping meta / non-task lines (run.py writes a meta header).""" + """Parse JSONL skipping meta / non-task lines.""" out: list[dict] = [] for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip(): @@ -57,6 +57,17 @@ def _eval_creds(monkeypatch): monkeypatch.delenv("REDIS_PORT", raising=False) +def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): + monkeypatch.setenv("SOME_SECRET", "x") + + environment = runner_live.stdio_server_env() + + assert "SOME_SECRET" not in environment + assert environment["PLANE_API_KEY"] == "test-key" + assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" + assert environment["PLANE_BASE_URL"] == "https://api.plane.so" + + # --------------------------------------------------------------------------- # Resume skip decision (pure) # --------------------------------------------------------------------------- @@ -87,13 +98,13 @@ def test_should_skip_resume_row_non_null_error_retries(): def test_load_resume_skip_keys_summary(tmp_path: Path): p = tmp_path / "out.jsonl" rows = [ - {"task_id": "R1", "rep": 0, "surface": "v2", "error": None, "error_class": None}, - {"task_id": "R1", "rep": 1, "surface": "v2", "error": "x", "error_class": "infra_seed"}, - {"task_id": "W1", "rep": 0, "surface": "v2", "error": None, "success": False}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, ] p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") - assert skip == {("R1", 0), ("W1", 0)} + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local"), ("W1", 0, "local")} assert n_skip == 2 assert n_retry == 1 @@ -102,21 +113,21 @@ def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path: Path): """Historical error row whose later row succeeded must not inflate n_retry.""" p = tmp_path / "out.jsonl" rows = [ - {"task_id": "R1", "rep": 0, "surface": "v2", "error": "boom", "error_class": "infra_cli"}, - {"task_id": "R1", "rep": 0, "surface": "v2", "error": None, "error_class": None, "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, ] p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") - assert skip == {("R1", 0)} + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} assert n_skip == 1 assert n_retry == 0 -def test_load_resume_skip_keys_surface_mismatch(tmp_path: Path): +def test_load_resume_skip_keys_label_mismatch(tmp_path: Path): p = tmp_path / "out.jsonl" - p.write_text(json.dumps({"task_id": "R1", "rep": 0, "surface": "full", "error": None}) + "\n") - with pytest.raises(SystemExit, match="surface"): - load_resume_skip_keys(p, surface="v2") + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") + with pytest.raises(SystemExit, match="label"): + load_resume_skip_keys(p, label="local") def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): @@ -126,7 +137,7 @@ def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): { "task_id": "R1", "rep": 0, - "surface": "v2", + "label": "local", "battery": "aaaaaaaaaaaa", "model": "sonnet", "driver": "claude-cli", @@ -137,16 +148,16 @@ def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): encoding="utf-8", ) with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, surface="v2", battery="bbbbbbbbbbbb") + load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="haiku") + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") with pytest.raises(SystemExit, match="driver"): - load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="sonnet", driver="sdk") + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") # Missing keys on older rows: pass (back-compat) p2 = tmp_path / "old.jsonl" - p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "error": None}) + "\n") - skip, _, _ = load_resume_skip_keys(p2, surface="v2", battery="anything", model="sonnet", driver="claude-cli") - assert ("R1", 0) in skip + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0, "local") in skip def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): @@ -156,7 +167,7 @@ def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): { "task_id": "R1", "rep": 0, - "surface": "v2", + "label": "local", "model": "provider-reported-id", "requested_model": "standard", "requested_tier": "standard", @@ -168,29 +179,29 @@ def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): encoding="utf-8", ) - skip, _, _ = load_resume_skip_keys(p, surface="v2", model="old-standard-id") - assert skip == {("R1", 0)} + skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") + assert skip == {("R1", 0, "local")} with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, surface="v2", model="new-standard-id") + load_resume_skip_keys(p, label="local", model="new-standard-id") def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): p = tmp_path / "out.jsonl" p.write_text( - json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "error": None}) + json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n" - + '{"task_id": "W1", "rep": 0, "surface": "v2", "error":\n', # truncated + + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated encoding="utf-8", ) - skip, n_skip, n_retry = load_resume_skip_keys(p, surface="v2") - assert skip == {("R1", 0)} + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} assert n_skip == 1 err = capsys.readouterr().err assert "invalid JSON" in err def test_load_resume_skip_keys_missing_file(tmp_path: Path): - skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", surface="v2") + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") assert skip == set() and n_skip == 0 and n_retry == 0 @@ -240,9 +251,10 @@ def boom_seed(plane, run_id, needs, ctx): [task], model_alias="standard", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", + resolved_model_id="sonnet", ) ) assert rc == 0 @@ -300,7 +312,7 @@ def run_task(self, *args, **kwargs): [task], model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", ) @@ -357,7 +369,7 @@ async def verify(*a, **k): [task], model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", ) @@ -421,7 +433,7 @@ async def verify(*a, **k): [task], model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", ) @@ -478,7 +490,7 @@ async def verify(*a, **k): [task], model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", ) @@ -662,12 +674,6 @@ def test_battery_fingerprint_stable_and_sensitive(): "optimal_tools": {"b", "a"}, "alternate_tools": {"c"}, "optimal_calls": 2, - "surface_tools": { - "v2": { - "optimal_tools": {"find_work_items"}, - "alternate_tools": set(), - } - }, } t2 = { "id": "B", @@ -675,7 +681,6 @@ def test_battery_fingerprint_stable_and_sensitive(): "optimal_tools": {"x"}, "alternate_tools": set(), "optimal_calls": 1, - "surface_tools": {}, } # Order of list must not matter (sorted by id). h1 = battery_fingerprint([t2, t1]) @@ -779,8 +784,7 @@ def test_print_table_shows_infra_errors(capsys): assert " 2" in out # i_err column value -def test_is_infra_error_row_covers_sdk(): - assert is_infra_error_row({"error_class": "infra_sdk"}) is True +def test_is_infra_error_row_covers_infrastructure_prefix(): assert is_infra_error_row({"error_class": "infra_cli"}) is True assert is_infra_error_row({"error_class": "task"}) is False @@ -788,8 +792,8 @@ def test_is_infra_error_row_covers_sdk(): def test_load_rows_dedupe_latest_wins(tmp_path: Path): p = tmp_path / "dup.jsonl" rows = [ - {"task_id": "R1", "rep": 0, "surface": "v2", "success": True, "num_calls": 1}, - {"task_id": "R1", "rep": 0, "surface": "v2", "success": False, "num_calls": 9}, + {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, ] p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") loaded = load_rows(p) # default dedupe=latest @@ -801,8 +805,8 @@ def test_load_rows_dedupe_latest_wins(tmp_path: Path): def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): p = tmp_path / "dup.jsonl" rows = [ - {"task_id": "R1", "rep": 0, "surface": "v2", "success": True}, - {"task_id": "R1", "rep": 0, "surface": "v2", "success": False}, + {"task_id": "R1", "rep": 0, "label": "local", "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False}, ] p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") loaded = load_rows(p, dedupe="none") @@ -851,7 +855,7 @@ async def correctly_fails(plane, ctx, run): "verify": always_ok, }, ] - rc = asyncio.run(run_canary(tasks, surface="full")) + rc = asyncio.run(run_canary(tasks, label="local")) assert rc == 1 @@ -878,25 +882,19 @@ async def reject(plane, ctx, run): "verify": reject, }, ] - rc = asyncio.run(run_canary(tasks, surface="full")) + rc = asyncio.run(run_canary(tasks, label="local")) assert rc == 0 def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): fake_plane = MagicMock() monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_canary, "seed", lambda *a, **k: None) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) monkeypatch.setattr( runner_canary, - "resolve_surface_tool_sets", - lambda task, surface: { - "skip": "unsupported on surface", - "optimal_tools": set(), - "alternate_tools": set(), - "classification": "exact", - }, + "seed", + lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) tasks = [ { "id": "SKIPME", @@ -908,7 +906,7 @@ def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): "verify": lambda *a, **k: (False, "unused"), }, ] - rc = asyncio.run(run_canary(tasks, surface="v2")) + rc = asyncio.run(run_canary(tasks, label="local")) assert rc == 1 @@ -958,7 +956,7 @@ async def verify_ok(plane, ctx, run): [task], model_alias="sonnet", reps=3, - surface="full", + label="local", out_path=out, driver_name="claude-cli", ) @@ -975,14 +973,14 @@ async def verify_ok(plane, ctx, run): def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): out = tmp_path / "resume.jsonl" - # Pre-write: completed R1/0 + infra R2/0 (same surface/battery/model/driver as this run). + # Pre-write: completed R1/0 + infra R2/0 (same label/battery/model/driver as this run). # Battery is computed from the task list below — seed the file after we know it, # or write rows without battery (back-compat) and only check skip/retry behavior. prior = [ { "task_id": "R1", "rep": 0, - "surface": "full", + "label": "local", "driver": "claude-cli", "model": "sonnet", "error": None, @@ -992,7 +990,7 @@ def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypat { "task_id": "R2", "rep": 0, - "surface": "full", + "label": "local", "driver": "claude-cli", "model": "sonnet", "error": "HttpError: 409", @@ -1058,7 +1056,7 @@ async def verify_ok(plane, ctx, run): tasks, model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=out, driver_name="claude-cli", resume=True, @@ -1075,3 +1073,56 @@ async def verify_ok(plane, ctx, run): assert new_r2["success"] is True assert new_r2["error_class"] is None assert new_r2["final_text"] == "done" + + +def test_task_skipped_from_seed_records_a_skip_row(tmp_path: Path, monkeypatch): + """A fixture that cannot be seeded records a skip — no agent, no crash. + + Genuine skips (an absent activity worker, a plan-gated feature) reach the + row through TaskSkipped, so the driver must never run and the row must not + count as a failure. + """ + out = tmp_path / "out.jsonl" + driven: list[str] = [] + torn: list[Any] = [] + + def skip_seed(*_args: Any, **_kwargs: Any) -> None: + raise TaskSkipped("env:no-activity-worker") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", skip_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: torn.append(1)) + monkeypatch.setattr( + runner_live, + "get_driver", + lambda *a, **k: driven.append("ran") or MagicMock(), + ) + + tasks = [ + { + "id": "L2", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": {"activity_feed"}, + "verify": None, # never reached + }, + ] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + assert driven == ["ran"] # driver is constructed once per run, never invoked + row = _data_rows(out)[0] + assert row["task_id"] == "L2" + assert row["skipped"] == "env:no-activity-worker" + assert row["error"] is None + assert row["error_class"] is None + assert row["label"] == "local" + assert torn == [1] # teardown still runs + + # `skipped` is the discriminator, not `success` — a skip must leave the + # success denominator empty rather than counting as a failed task. + summary = summarize(load_rows(out)) + assert "L2" not in summary + assert summary["_meta"]["aggregate_n"] == 0 diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index 0a098338..e1167b06 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -10,6 +10,8 @@ import pytest +from evals.cli import main as eval_main +from evals.cli import resolve_model_for_driver from evals.drivers import ( KNOWN_DRIVERS, AntigravityCliDriver, @@ -41,8 +43,6 @@ write_all_fd, ) from evals.proxy import main as proxy_main -from evals.run import main as eval_main -from evals.run import resolve_model_for_driver REPO = Path(__file__).resolve().parent.parent @@ -1103,7 +1103,7 @@ def fake_run(cmd, **kwargs): "use_proxy": True, "record_result_payloads": True, "python_bin": sys.executable, - "server_command": ["/ext/bin/foreign-mcp", "stdio", "--v2"], + "server_command": ["/ext/bin/foreign-mcp", "stdio", "--mode", "candidate"], } if Driver is CodexCliDriver: kwargs["allow_live"] = True @@ -1142,7 +1142,6 @@ def test_model_tiers_resolve_per_driver_and_provider(): assert resolve_model_for_driver("api", "fast", provider="anthropic") == "claude-haiku-4-5" assert resolve_model_for_driver("api", "standard", provider="openai") == "gpt-5.6-sol" assert resolve_model_for_driver("api", "fast", provider="openai") == "gpt-5.6-luna" - assert resolve_model_for_driver("sdk", "standard", provider="openai") == "gpt-5.6-sol" assert resolve_model_for_driver("claude-cli", "standard") == "sonnet" assert resolve_model_for_driver("claude-cli", "fast") == "haiku" assert resolve_model_for_driver("codex-cli", "standard") == "gpt-5.6-sol" @@ -1879,7 +1878,7 @@ async def _verify(*a, **k): [task], model_alias="sonnet", reps=1, - surface="full", + label="local", out_path=tmp_path / "o.jsonl", driver_name="opencode-cli", server_cmd=["/bin/foreign", "stdio"], diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index b41a0ac2..fe889ec1 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -27,7 +27,7 @@ wilson_interval, ) from evals.results import RESULT_SCHEMA_VERSION, CallRecord, TaskResult, Usage -from evals.run import ( +from evals.runner import ( is_meta_or_non_task_row, load_resume_skip_keys, make_run_meta_row, @@ -94,7 +94,7 @@ def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): { "row_type": "meta", "run_id": "abc", - "surface": "v2", + "label": "candidate", "battery": "deadbeef0001", "model": "sonnet", "driver": "claude-cli", @@ -102,8 +102,8 @@ def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): "ts": "t", } ), - json.dumps({"surface": "v2", "rep": 0, "success": True}), # no task_id - json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "success": True, "num_calls": 2}), + json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), ] p.write_text("\n".join(lines) + "\n", encoding="utf-8") rows = load_rows(p) @@ -115,6 +115,8 @@ def test_task_result_schema_round_trip_owns_usage_shape(): result = TaskResult( row_type="result", task_id="R1", + label="local", + server="local", calls=[ CallRecord( tool="find_work_items", @@ -131,6 +133,8 @@ def test_task_result_schema_round_trip_owns_usage_shape(): row = result.to_row() assert row["schema_version"] == RESULT_SCHEMA_VERSION assert row["row_type"] == "result" + assert row["label"] == "local" + assert row["server"] == "local" assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] loaded = TaskResult.from_row(row) assert loaded.row_type == "result" @@ -167,9 +171,9 @@ def test_real_historical_rows_parse_and_report_with_backward_defaults(): def test_dedupe_rows_latest_pure(): rows = [ - {"task_id": "R1", "rep": 0, "surface": "full", "num_calls": 1}, - {"task_id": "R1", "rep": 0, "surface": "full", "num_calls": 5}, - {"task_id": "R2", "rep": 0, "surface": "full", "num_calls": 3}, + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 5}, + {"task_id": "R2", "rep": 0, "label": "local", "num_calls": 3}, ] out = dedupe_rows_latest(rows) assert len(out) == 2 @@ -213,7 +217,7 @@ def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_pa { "task_id": task_id, "rep": rep, - "surface": "full", + "label": "local", "success": success, "num_calls": rep + 1, "calls": [], @@ -249,7 +253,7 @@ def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_pa def test_single_rep_summary_rendering_is_unchanged(capsys): - rows = [{"task_id": "R1", "rep": 0, "surface": "full", "success": True, "num_calls": 2, "calls": []}] + rows = [{"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 2, "calls": []}] report_mod.print_table(summarize(rows), "Summary: sample.jsonl") @@ -375,21 +379,21 @@ def _synth_row( num_calls: int = 2, alt: int | None = 0, oos: int | None = 0, - classification: str = "exact", + server: str = "local", skipped: str | None = None, error: str | None = None, error_class: str | None = None, - surface: str = "v2", + label: str = "local", ) -> dict[str, Any]: return { "task_id": tid, "rep": rep, - "surface": surface, + "label": label, "success": success, "num_calls": num_calls, "alternate_calls": alt, "out_of_set_calls": oos, - "classification": classification, + "server": server, "skipped": skipped, "error": error, "error_class": error_class, @@ -405,34 +409,34 @@ def test_format_surface_cell_variants(): assert format_surface_cell(_synth_row("R1", success=True, num_calls=3, alt=0, oos=0)) == "✅ 3c" assert format_surface_cell(_synth_row("R1", success=False, num_calls=4, alt=1, oos=1)) == "❌ 4c/2mp" # external: no mispick suffix - assert format_surface_cell(_synth_row("R1", classification="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" + assert format_surface_cell(_synth_row("R1", server="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" def test_multi_surface_table_snapshot_with_external(): - legacy = [ - _synth_row("R1", surface="full", num_calls=4, alt=1, oos=0), - _synth_row("R2", surface="full", success=False, num_calls=2), + local = [ + _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), + _synth_row("R2", label="local", success=False, num_calls=2), ] - v2 = [ - _synth_row("R1", surface="v2", num_calls=2, alt=0, oos=0), - _synth_row("R2", surface="v2", skipped="unsupported", num_calls=0), + candidate = [ + _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), + _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), ] external = [ - _synth_row("R1", surface="akhil", classification="external", alt=None, oos=None, num_calls=3), - _synth_row("R2", surface="akhil", classification="external", alt=None, oos=None, num_calls=1, success=False), - _synth_row("R3", surface="akhil", classification="external", error="timeout", error_class="infra_cli"), + _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), + _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), + _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), ] - table = build_multi_surface_table([("full", legacy), ("v2", v2), ("akhil", external)]) - assert table["columns"] == ["full", "v2", "akhil"] + table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) + assert table["columns"] == ["local", "candidate", "akhil"] assert "R1" in table["task_ids"] and "R3" in table["task_ids"] - assert table["cells"]["R1"]["full"] == "✅ 4c/1mp" - assert table["cells"]["R1"]["v2"] == "✅ 2c" + assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" + assert table["cells"]["R1"]["candidate"] == "✅ 2c" assert table["cells"]["R1"]["akhil"] == "✅ 3c" - assert table["cells"]["R2"]["v2"] == "skip" + assert table["cells"]["R2"]["candidate"] == "skip" assert table["cells"]["R3"]["akhil"] == "ERR" text = render_multi_surface_table(table, markdown=False) - assert "full" in text and "v2" in text and "akhil" in text + assert "local" in text and "candidate" in text and "akhil" in text assert "✅ 3c" in text assert "skip" in text assert "ERR" in text @@ -446,44 +450,44 @@ def test_multi_surface_table_snapshot_with_external(): # Footer: external mispicks n/a assert table["footer"]["akhil"]["mispicks"] is None - assert table["footer"]["full"]["mispicks"] == 1 + assert table["footer"]["local"]["mispicks"] == 1 assert table["footer"]["akhil"]["infra_errors"] == 1 def test_multi_surface_table_aggregates_reps_and_flags_unstable(): rows = [ - _synth_row("R1", rep=0, success=True, num_calls=2, surface="full"), - _synth_row("R1", rep=1, success=True, num_calls=3, surface="full"), - _synth_row("R1", rep=2, success=True, num_calls=2, surface="full"), - _synth_row("R2", rep=0, success=True, num_calls=1, surface="full"), - _synth_row("R2", rep=1, success=False, num_calls=4, surface="full"), - _synth_row("R2", rep=2, success=True, num_calls=2, surface="full"), + _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), + _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), + _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), + _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), + _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), + _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), ] - table = build_multi_surface_table([("full", rows)]) + table = build_multi_surface_table([("local", rows)]) assert table["multi_rep"] is True - assert table["cells"]["R1"]["full"] == "✅ 3/3 [0.44,1.00] 2-3c" - assert table["cells"]["R2"]["full"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" - assert table["footer"]["full"]["success"] == 5 - assert table["footer"]["full"]["n"] == 6 - assert table["footer"]["full"]["unstable_tasks"] == 1 + assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" + assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" + assert table["footer"]["local"]["success"] == 5 + assert table["footer"]["local"]["n"] == 6 + assert table["footer"]["local"]["unstable_tasks"] == 1 rendered = render_multi_surface_table(table) assert "measured noise floor: 1 task flipped at least once" in rendered assert "minimum meaningful difference: 2 tasks" in rendered def test_single_rep_multi_surface_rendering_is_unchanged(): - rows = [_synth_row("R1", surface="full", success=True, num_calls=2)] + rows = [_synth_row("R1", label="local", success=True, num_calls=2)] - rendered = render_multi_surface_table(build_multi_surface_table([("full", rows)])) + rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) assert rendered == ( - "task what full \n" + "task what local \n" "-------------------------------------------------------\n" "R1 In project P, what is the curren… ✅ 2c\n" "-------------------------------------------------------\n" - "full success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" + "local success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" ) @@ -491,17 +495,17 @@ def test_report_main_table_cli(tmp_path: Path, capsys): f1 = tmp_path / "a.jsonl" f2 = tmp_path / "b.jsonl" f1.write_text( - json.dumps(_synth_row("R1", surface="full", num_calls=2)) + "\n", + json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", encoding="utf-8", ) f2.write_text( - json.dumps(_synth_row("R1", surface="v2", num_calls=1)) + "\n", + json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8", ) rc = report_mod.main(["--table", str(f1), str(f2)]) assert rc == 0 out = capsys.readouterr().out - assert "full" in out and "v2" in out + assert "local" in out and "candidate" in out assert "R1" in out @@ -509,11 +513,11 @@ def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path f1 = tmp_path / "old.jsonl" f2 = tmp_path / "new.jsonl" f1.write_text( - json.dumps({**_synth_row("R1", surface="full"), "battery": "6425dcc64404"}) + "\n", + json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", encoding="utf-8", ) f2.write_text( - json.dumps({**_synth_row("R1", surface="v2"), "battery": "newfinger001"}) + "\n", + json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", encoding="utf-8", ) @@ -527,7 +531,7 @@ def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path def test_report_main_markdown_flag(tmp_path: Path, capsys): f1 = tmp_path / "a.jsonl" - f1.write_text(json.dumps(_synth_row("R1", surface="v2", num_calls=1)) + "\n", encoding="utf-8") + f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") rc = report_mod.main(["--table", "--markdown", str(f1)]) assert rc == 0 out = capsys.readouterr().out @@ -539,10 +543,10 @@ def test_report_main_markdown_flag(tmp_path: Path, capsys): def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): p = tmp_path / "d.jsonl" rows = [ - _synth_row("R1", surface="full", num_calls=1, success=True), - {**_synth_row("R1", surface="full", num_calls=9, success=False)}, + _synth_row("R1", label="local", num_calls=1, success=True), + {**_synth_row("R1", label="local", num_calls=9, success=False)}, ] - # Both rows same (task_id, rep, surface) — latest-wins would keep one. + # Both rows have the same (task_id, rep, label), so latest-wins keeps one. p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") rc = report_mod.main(["--no-dedupe", str(p)]) assert rc == 0 @@ -554,7 +558,7 @@ def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): # --------------------------------------------------------------------------- -# Meta line (run.py) +# Meta line # --------------------------------------------------------------------------- @@ -562,7 +566,8 @@ def test_make_run_meta_row_and_write_once(tmp_path: Path): path = tmp_path / "out.jsonl" meta = make_run_meta_row( run_id="rid", - surface="v2", + label="candidate", + server="local", battery="abcd1234ef00", model="sonnet", driver="claude-cli", @@ -575,7 +580,7 @@ def test_make_run_meta_row_and_write_once(tmp_path: Path): assert maybe_write_run_meta(path, meta) is True # Append a data row — a truncating rewrite on the second call would destroy it. with path.open("a", encoding="utf-8") as fh: - fh.write(json.dumps({"task_id": "R1", "rep": 0, "surface": "v2", "success": True}) + "\n") + fh.write(json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True}) + "\n") assert maybe_write_run_meta(path, meta) is False # file non-empty lines = path.read_text(encoding="utf-8").splitlines() assert len(lines) == 2 @@ -591,7 +596,7 @@ def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): json.dumps( { "row_type": "meta", - "surface": "v2", + "label": "candidate", "battery": "bbbbbbbbbbbb", "model": "sonnet", "driver": "claude-cli", @@ -601,7 +606,7 @@ def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): { "task_id": "R1", "rep": 0, - "surface": "v2", + "label": "candidate", "error": None, "error_class": None, "success": True, @@ -613,13 +618,13 @@ def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): encoding="utf-8", ) skip, n_skip, n_retry = load_resume_skip_keys( - p, surface="v2", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" ) - assert skip == {("R1", 0)} + assert skip == {("R1", 0, "candidate")} assert n_skip == 1 and n_retry == 0 with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, surface="v2", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") + load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") # --------------------------------------------------------------------------- diff --git a/tests/test_evals_surface.py b/tests/test_evals_surface.py deleted file mode 100644 index 5f85c82e..00000000 --- a/tests/test_evals_surface.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Offline tests for eval --surface plumbing and classification overlays.""" - -from __future__ import annotations - -import pytest - -from evals.run import KNOWN_SURFACES, classify_call, parse_args, stdio_server_env -from evals.tasks import TASKS_BY_ID, resolve_surface_tool_sets - - -@pytest.fixture(autouse=True) -def _eval_creds(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - -def test_known_surfaces(): - assert KNOWN_SURFACES == {"full", "v2", "v2-schema"} - - -def test_stdio_env_full_does_not_set_surface(monkeypatch): - env = stdio_server_env(surface="full") - assert "PLANE_MCP_SURFACE" not in env - assert env["PLANE_API_KEY"] == "test-key" - assert env["PLANE_WORKSPACE_SLUG"] == "test-ws" - assert env["PLANE_BASE_URL"] == "https://api.plane.so" - # Never inherits ambient secrets - monkeypatch.setenv("SOME_SECRET", "x") - env2 = stdio_server_env(surface="full") - assert "SOME_SECRET" not in env2 - - -def test_stdio_env_v2_sets_plane_mcp_surface(): - env = stdio_server_env(surface="v2") - assert env["PLANE_MCP_SURFACE"] == "v2" - assert env["PLANE_API_KEY"] == "test-key" - - -def test_stdio_env_v2_schema_sets_plane_mcp_surface(): - env = stdio_server_env(surface="v2-schema") - assert env["PLANE_MCP_SURFACE"] == "v2-schema" - - -def test_parse_args_accepts_v2_and_full(): - a = parse_args(["--surface", "v2", "--dry-run"]) - assert a.surface == "v2" - b = parse_args(["--surface", "full"]) - assert b.surface == "full" - - -def test_r1_v2_overlay_exact_find_work_items(): - r1 = TASKS_BY_ID["R1"] - full = resolve_surface_tool_sets(r1, "full") - assert full["classification"] == "exact" - assert full["skip"] is None - assert "list_work_items" in full["optimal_tools"] - - v2 = resolve_surface_tool_sets(r1, "v2") - assert v2["classification"] == "exact" - assert v2["skip"] is None - assert v2["optimal_tools"] == {"find_work_items"} - assert "list_work_items" not in v2["optimal_tools"] - - -def test_w1_v2_overlay(): - w1 = TASKS_BY_ID["W1"] - v2 = resolve_surface_tool_sets(w1, "v2") - assert v2["classification"] == "exact" - assert "create_work_item" in v2["optimal_tools"] - assert "get_workspace_context" in v2["optimal_tools"] - assert v2["optimal_calls"] == 2 - - -def test_s1_v2_unsupported_skip(): - s1 = TASKS_BY_ID["S1"] - v2 = resolve_surface_tool_sets(s1, "v2") - assert v2["skip"] is not None - assert "schema" in v2["skip"].lower() or "property" in v2["skip"].lower() or "not on the" in v2["skip"] - assert v2["classification"] == "exact" - - full = resolve_surface_tool_sets(s1, "full") - assert full["skip"] is None - assert "create_work_item_property" in full["optimal_tools"] - - -def test_s1_v2_schema_overlay(): - s1 = TASKS_BY_ID["S1"] - out = resolve_surface_tool_sets(s1, "v2-schema") - assert out["skip"] is None - assert out["classification"] == "exact" - assert out["optimal_tools"] == {"resolve_work_item_type", "create_work_item_property"} - assert out["optimal_calls"] == 2 - - -def test_unknown_surface_without_overlay_is_approximate(): - """A surface with no overlay falls back to flat sets + approximate.""" - r1 = TASKS_BY_ID["R1"] - # Fabricate: use a surface name that has no overlay - out = resolve_surface_tool_sets(r1, "experimental") - assert out["classification"] == "approximate" - assert out["skip"] is None - assert out["optimal_tools"] == set(r1["optimal_tools"]) - - -def test_classify_uses_resolved_sets(): - v2 = resolve_surface_tool_sets(TASKS_BY_ID["R1"], "v2") - assert classify_call("find_work_items", v2["optimal_tools"], v2["alternate_tools"]) == "optimal" - assert classify_call("list_work_items", v2["optimal_tools"], v2["alternate_tools"]) == "out_of_set" - assert classify_call("get_work_item", v2["optimal_tools"], v2["alternate_tools"]) == "alternate" - - -def test_skip_path_no_network(monkeypatch): - """Unsupported surface skip must not call seed/teardown/agent.""" - from evals.runner import live as run_mod - - seeded = [] - torn = [] - - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr( - run_mod, - "seed", - lambda *a, **k: seeded.append(1) or (_ for _ in ()).throw(AssertionError("seed should not run")), - ) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: torn.append(1)) - - import asyncio - import tempfile - from pathlib import Path - - with tempfile.TemporaryDirectory() as td: - out = Path(td) / "out.jsonl" - rc = asyncio.run( - run_mod.run_live( - [TASKS_BY_ID["S1"]], - model_alias="sonnet", - reps=1, - surface="v2", - out_path=out, - ) - ) - assert rc == 0 - assert seeded == [] - text = out.read_text(encoding="utf-8") - assert "S1" in text - assert "skipped" in text - # First line may be meta header; pick the task row. - rows = [__import__("json").loads(ln) for ln in text.strip().splitlines() if ln.strip()] - row = next(r for r in rows if r.get("task_id") == "S1") - assert row["surface"] == "v2" - assert row["skipped"] - assert row["classification"] == "exact" From 0700a9679de40a9b22a06e00f0afbbf3395f8cbb Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 16:50:40 +0530 Subject: [PATCH 17/93] Give the CLI drivers the same package shape as the API driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals/drivers held two kinds of driver in two different shapes: api/ was a package (one driver plus its per-provider backends) while the CLI side was eight loose files at the root, despite being the same shape — one template plus its per-vendor implementations plus shared subprocess and sidecar machinery. The CLI drivers now live in drivers/cli/, so each kind is a package holding its own machinery and the root keeps only what both kinds use: the driver protocol and the shared token estimate. base.py becomes protocol.py, which is what it holds and is a name we allow. Imports through `evals.drivers` are unchanged; only the modules behind that face moved. --- evals/drivers/__init__.py | 59 ++++++++++----------- evals/drivers/api/driver.py | 2 +- evals/drivers/cli/__init__.py | 64 +++++++++++++++++++++++ evals/drivers/{ => cli}/antigravity.py | 2 +- evals/drivers/{ => cli}/claude.py | 4 +- evals/drivers/{ => cli}/codex.py | 6 +-- evals/drivers/{ => cli}/opencode.py | 2 +- evals/drivers/{ => cli}/process.py | 0 evals/drivers/{ => cli}/sidecar.py | 2 +- evals/drivers/{cli.py => cli/template.py} | 6 +-- evals/drivers/{base.py => protocol.py} | 0 tests/test_evals_proxy.py | 4 +- 12 files changed, 106 insertions(+), 45 deletions(-) create mode 100644 evals/drivers/cli/__init__.py rename evals/drivers/{ => cli}/antigravity.py (99%) rename evals/drivers/{ => cli}/claude.py (98%) rename evals/drivers/{ => cli}/codex.py (98%) rename evals/drivers/{ => cli}/opencode.py (98%) rename evals/drivers/{ => cli}/process.py (100%) rename evals/drivers/{ => cli}/sidecar.py (99%) rename evals/drivers/{cli.py => cli/template.py} (98%) rename evals/drivers/{base.py => protocol.py} (100%) diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index daf2c4ee..97ede1fc 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -29,7 +29,7 @@ ``response_item`` / ``function_call`` payloads (name + arguments JSON string) - Marked **experimental**; live runs are opt-in (metered quota). -This package splits that surface into focused modules (base types, subprocess +This package splits that surface into focused modules (protocol types, subprocess lifecycle, recording-proxy glue, and per-vendor drivers). Import from ``evals.drivers`` as before — public names are re-exported here. """ @@ -38,13 +38,37 @@ from typing import Any -from evals.drivers.antigravity import ( +from evals.drivers.api import ApiDriver +from evals.drivers.cli import ( AntigravityCliDriver, + ClaudeCliDriver, + CliDriver, + CodexCliDriver, + OpencodeCliDriver, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + find_claude_transcript, + find_codex_rollout, + harvest_proxy_after_cli_timeout, + kill_process_group, + load_proxy_sidecar, + load_proxy_sidecar_calls, + normalize_claude_usage, + note_timeout_kill, + parse_claude_json_result, + parse_claude_transcript_calls, + parse_codex_jsonl_events, + parse_codex_rollout_calls, prepare_antigravity_fake_home, + proxy_wrap_server_command, + run_cli_subprocess, + wait_for_proxy_meta, write_antigravity_mcp_config, + write_claude_mcp_config, + write_codex_mcp_override_args, + write_opencode_mcp_config, ) -from evals.drivers.api import ApiDriver -from evals.drivers.base import ( +from evals.drivers.protocol import ( REPO_ROOT, AgentDriver, AgentRun, @@ -55,33 +79,6 @@ split_plane_and_client_calls, strip_mcp_prefix, ) -from evals.drivers.claude import ( - ClaudeCliDriver, - find_claude_transcript, - normalize_claude_usage, - parse_claude_json_result, - parse_claude_transcript_calls, - write_claude_mcp_config, -) -from evals.drivers.cli import CliDriver -from evals.drivers.codex import ( - CodexCliDriver, - find_codex_rollout, - parse_codex_jsonl_events, - parse_codex_rollout_calls, - write_codex_mcp_override_args, -) -from evals.drivers.opencode import OpencodeCliDriver, write_opencode_mcp_config -from evals.drivers.process import kill_process_group, note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - load_proxy_sidecar, - load_proxy_sidecar_calls, - proxy_wrap_server_command, - wait_for_proxy_meta, -) from evals.results import CallRecord, TaskResult # Registry diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 85a27b94..af5387d9 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -23,7 +23,7 @@ ToolSpec, create_backend, ) -from evals.drivers.base import AgentRun +from evals.drivers.protocol import AgentRun from evals.drivers.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens from evals.results import Usage diff --git a/evals/drivers/cli/__init__.py b/evals/drivers/cli/__init__.py new file mode 100644 index 00000000..90fda0fc --- /dev/null +++ b/evals/drivers/cli/__init__.py @@ -0,0 +1,64 @@ +"""CLI agent drivers and their shared execution machinery.""" + +from evals.drivers.cli.antigravity import ( + AntigravityCliDriver, + prepare_antigravity_fake_home, + write_antigravity_mcp_config, +) +from evals.drivers.cli.claude import ( + ClaudeCliDriver, + find_claude_transcript, + normalize_claude_usage, + parse_claude_json_result, + parse_claude_transcript_calls, + write_claude_mcp_config, +) +from evals.drivers.cli.codex import ( + CodexCliDriver, + find_codex_rollout, + parse_codex_jsonl_events, + parse_codex_rollout_calls, + write_codex_mcp_override_args, +) +from evals.drivers.cli.opencode import OpencodeCliDriver, write_opencode_mcp_config +from evals.drivers.cli.process import kill_process_group, note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_wrap_server_command, + wait_for_proxy_meta, +) +from evals.drivers.cli.template import CliDriver + +__all__ = [ + "AntigravityCliDriver", + "ClaudeCliDriver", + "CliDriver", + "CodexCliDriver", + "OpencodeCliDriver", + "apply_proxy_sidecar", + "ensure_proxy_pythonpath", + "find_claude_transcript", + "find_codex_rollout", + "harvest_proxy_after_cli_timeout", + "kill_process_group", + "load_proxy_sidecar", + "load_proxy_sidecar_calls", + "normalize_claude_usage", + "note_timeout_kill", + "parse_claude_json_result", + "parse_claude_transcript_calls", + "parse_codex_jsonl_events", + "parse_codex_rollout_calls", + "prepare_antigravity_fake_home", + "proxy_wrap_server_command", + "run_cli_subprocess", + "wait_for_proxy_meta", + "write_antigravity_mcp_config", + "write_claude_mcp_config", + "write_codex_mcp_override_args", + "write_opencode_mcp_config", +] diff --git a/evals/drivers/antigravity.py b/evals/drivers/cli/antigravity.py similarity index 99% rename from evals/drivers/antigravity.py rename to evals/drivers/cli/antigravity.py index 8043d544..bf8a786c 100644 --- a/evals/drivers/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -8,7 +8,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.cli import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput # Antigravity CLI (agy) — proxy-first # --------------------------------------------------------------------------- diff --git a/evals/drivers/claude.py b/evals/drivers/cli/claude.py similarity index 98% rename from evals/drivers/claude.py rename to evals/drivers/cli/claude.py index 82d566fd..c7b7472f 100644 --- a/evals/drivers/claude.py +++ b/evals/drivers/cli/claude.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any -from evals.drivers.base import normalize_tool_call, split_plane_and_client_calls -from evals.drivers.cli import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.protocol import normalize_tool_call, split_plane_and_client_calls def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: diff --git a/evals/drivers/codex.py b/evals/drivers/cli/codex.py similarity index 98% rename from evals/drivers/codex.py rename to evals/drivers/cli/codex.py index 28d99a46..2b855531 100644 --- a/evals/drivers/codex.py +++ b/evals/drivers/cli/codex.py @@ -8,9 +8,9 @@ from pathlib import Path from typing import Any -from evals.drivers.base import normalize_tool_call, split_plane_and_client_calls -from evals.drivers.cli import CliDriver, CliLaunch, CliOutput -from evals.drivers.process import run_cli_subprocess +from evals.drivers.cli.process import run_cli_subprocess +from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput +from evals.drivers.protocol import normalize_tool_call, split_plane_and_client_calls def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: diff --git a/evals/drivers/opencode.py b/evals/drivers/cli/opencode.py similarity index 98% rename from evals/drivers/opencode.py rename to evals/drivers/cli/opencode.py index 83b13865..b8402ba8 100644 --- a/evals/drivers/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -7,7 +7,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.cli import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput # OpenCode CLI — proxy-first # --------------------------------------------------------------------------- diff --git a/evals/drivers/process.py b/evals/drivers/cli/process.py similarity index 100% rename from evals/drivers/process.py rename to evals/drivers/cli/process.py diff --git a/evals/drivers/sidecar.py b/evals/drivers/cli/sidecar.py similarity index 99% rename from evals/drivers/sidecar.py rename to evals/drivers/cli/sidecar.py index 2a2d1b03..14ae6b1a 100644 --- a/evals/drivers/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from evals.drivers.base import REPO_ROOT +from evals.drivers.protocol import REPO_ROOT def proxy_wrap_server_command( diff --git a/evals/drivers/cli.py b/evals/drivers/cli/template.py similarity index 98% rename from evals/drivers/cli.py rename to evals/drivers/cli/template.py index 74a28684..7f624d69 100644 --- a/evals/drivers/cli.py +++ b/evals/drivers/cli/template.py @@ -12,14 +12,14 @@ from pathlib import Path from typing import Any -from evals.drivers.base import REPO_ROOT, AgentRun -from evals.drivers.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.sidecar import ( +from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( apply_proxy_sidecar, ensure_proxy_pythonpath, harvest_proxy_after_cli_timeout, proxy_wrap_server_command, ) +from evals.drivers.protocol import REPO_ROOT, AgentRun @dataclass diff --git a/evals/drivers/base.py b/evals/drivers/protocol.py similarity index 100% rename from evals/drivers/base.py rename to evals/drivers/protocol.py diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index e1167b06..7427f97a 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -31,7 +31,7 @@ write_antigravity_mcp_config, write_opencode_mcp_config, ) -from evals.drivers.cli import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput from evals.drivers.token_counting import estimate_result_tokens from evals.proxy import ( SHUTDOWN_DEADLINE_S, @@ -580,7 +580,7 @@ def test_agent_run_to_harness_propagates_proxy_fields(): def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.cli.time.perf_counter", lambda: clock["now"]) + monkeypatch.setattr("evals.drivers.cli.template.time.perf_counter", lambda: clock["now"]) class MinimalCliDriver(CliDriver): name = "minimal-cli" From 22ddca44ca1dae22c64ab69082302cb8e418b35d Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 16:50:48 +0530 Subject: [PATCH 18/93] Stop treating a local Plane instance as an eval prerequisite The README opened by telling you to export PLANE_EE_API_DIR and PLANE_EE_VENV and run evals/env.sh, which made a plane-ee checkout and a second virtualenv look mandatory. They never were. The harness reaches its target through three EVAL_PLANE_* variables and knows nothing else about it, so a local plane-ee, a staging box, and a hosted workspace are the same thing to it. Those two variables were prerequisites of the bootstrap script alone. How a developer gets an instance to measure against is their own setup, so the script and the mock feature-flag server leave the repo; localdev/ is ignored. The prerequisites are now what they actually are: a reachable Plane with a key that can create and delete fixtures, and model access for the chosen driver. The local gotchas stay, including the feature-flag cache trap, which is the most expensive thing here to rediscover. --- .gitignore | 6 +- evals/DESIGN.md | 22 ++-- evals/README.md | 57 +++++---- evals/env.sh | 282 -------------------------------------------- evals/mock_flags.py | 120 ------------------- 5 files changed, 54 insertions(+), 433 deletions(-) delete mode 100755 evals/env.sh delete mode 100644 evals/mock_flags.py diff --git a/.gitignore b/.gitignore index fee4f34a..f6ee0cf7 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,7 @@ htmlcov/ .tox/ .hypothesis/ -# Eval harness output + local env bootstrap state +# Eval harness output evals/output/ # Keep earlier local runs ignored after the default directory rename. evals/results/ @@ -52,6 +52,10 @@ evals/.env-pids evals/.api_runserver.log evals/.mock_flags.log +# Booting a local Plane to run evals against is each developer's own setup, +# not part of this repo. The harness itself only needs the EVAL_PLANE_* vars. +localdev/ + # Mypy .mypy_cache/ .dmypy.json diff --git a/evals/DESIGN.md b/evals/DESIGN.md index edfbc970..eb27b6b8 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -209,24 +209,32 @@ evals/ debias.py I1-I5 and L1-L5 tasks and verifiers drivers/ __init__.py public exports and driver registry - base.py AgentDriver, AgentRun, normalization, and common row mapping + protocol.py AgentDriver, AgentRun, normalization, and common row mapping token_counting.py tool-result token sizing - claude.py Claude Code CLI driver - codex.py Codex CLI driver - antigravity.py Antigravity CLI driver - opencode.py opencode CLI driver - process.py shared subprocess lifecycle - sidecar.py recording-proxy command and sidecar handling api/ backend.py neutral backend protocol and turn/tool dataclasses driver.py provider-neutral MCP/model loop anthropic.py Anthropic Messages translation openai.py OpenAI Chat Completions translation + cli/ + template.py shared CLI driver template + process.py subprocess lifecycle + sidecar.py recording-proxy command and sidecar handling + claude.py Claude Code CLI driver + codex.py Codex CLI driver + antigravity.py Antigravity CLI driver + opencode.py OpenCode CLI driver proxy.py stdlib-only JSON-RPC recording relay seed/ Plane fixture creation and teardown report/ summaries, A/B comparison, and multi-surface tables + ``` +Booting a Plane instance to measure against is deliberately outside this tree. The +harness reaches its target through three `EVAL_PLANE_*` variables and knows nothing +else about how that instance runs, so a local plane-ee, a shared staging box, and a +hosted workspace are the same thing to it. + The stable import and command surfaces are intentional: `from evals.tasks import ...`, `from evals.drivers import ...`, and `python -m evals` remain the public boundaries even though their implementations are split across packages and focused modules. diff --git a/evals/README.md b/evals/README.md index fd145a5d..b81e14d9 100644 --- a/evals/README.md +++ b/evals/README.md @@ -16,26 +16,18 @@ agent's final text. ## Prerequisites -1. **A Plane API endpoint.** For local runs, `evals/env.sh` starts plane-ee on `:8000` plus - a mock feature-flag server on `:9911` that turns every discovered flag on. +1. **A reachable Plane instance and API key.** Any reachable instance works, local or + hosted. The key must be able to create and delete the catalog's project and + workspace-scoped fixtures. Genuine plan or feature gates are recorded as skips where + the fixture code handles them. Configure the harness with exactly these three values: ```bash - export PLANE_EE_API_DIR=/path/to/plane-ee/apps/api - export PLANE_EE_VENV=/path/to/plane-ee-venv - evals/env.sh up # down | status + export EVAL_PLANE_BASE_URL=https://your-plane.example.com + export EVAL_PLANE_WORKSPACE_SLUG=your-workspace-slug + export EVAL_PLANE_API_KEY=plane_api_your_key ``` -2. **A workspace and an API key** on that instance. The key must be able to create and - delete the catalog's project and workspace-scoped fixtures. Genuine plan or feature - gates are recorded as skips where the fixture code handles them. - - ```bash - export EVAL_PLANE_BASE_URL=http://localhost:8000 - export EVAL_PLANE_WORKSPACE_SLUG= - export EVAL_PLANE_API_KEY=plane_api_... - ``` - -3. **Model access for the driver you pick.** The API driver uses +2. **Model access for the driver you pick.** The API driver uses `ANTHROPIC_API_KEY` by default; OpenAI requires its SDK and `OPENAI_API_KEY`. CLI drivers require their corresponding local CLI to already be authenticated. @@ -155,8 +147,13 @@ Tasks that touch **workspace-scoped** fixtures (release tags, customer propertie if two runs share a workspace. Give each concurrent run its own workspace: ```bash -EVAL_PLANE_WORKSPACE_SLUG=ws1 ... --label local --out evals/output/local.jsonl & -EVAL_PLANE_WORKSPACE_SLUG=ws2 ... --label their-pr --out evals/output/their-pr.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws1 .venv/bin/python -m evals \ + --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --out evals/output/local.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws2 .venv/bin/python -m evals \ + --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ + --out evals/output/their-pr.jsonl & wait ``` @@ -228,6 +225,19 @@ tasks, fixtures, or verifiers. `cycles_open_past` fixture variant because it asks the agent to close Sprint 12; the seeder must not pre-close the cycle that the task is meant to change. +## Running a local Plane + +Any reachable Plane works, so how you get one is your own setup and is not kept in this +repo. If you run plane-ee locally, two things make it usable for evals: + +- Point `FEATURE_FLAG_SERVER_BASE_URL` at a flag server that answers every flag as on. + The gated tasks (releases, customers, worklogs, work item types) need it, and the + hosted flag server has them off for a local workspace. +- Raise `API_KEY_RATE_LIMIT`; a full battery makes far more API calls than the default + allows. + +Keep such scripts outside version control — `localdev/` is ignored for exactly this. + ## Local gotchas - If seeded comments do not materialize as activities, the activity-feed task self-skips @@ -236,11 +246,12 @@ must not pre-close the cycle that the task is meant to change. cached per workspace, and the cache does not record which flag server answered. Any process that touches the DB while pointed at a *different* flag server than the running API — sourcing `plane-ee/apps/api/.env` gets you the remote one, where these flags are - off — caches that answer for the workspace, and the API then serves the cached miss - instead of asking its own mock. The tell is a canary that reports every verifier broken - at once. Clear `ff::*` and rotate `ff_ver:` (`plane.payment.flags.cache`) - and the next request refetches. Observed while creating a workspace from a Django - shell; a plan/licence problem looks identical from the outside, so check this first. + off — caches that answer for the workspace, and + the API then serves the cached miss instead of asking its own mock. The tell is a canary + that reports every verifier broken at once. Clear `ff::*` and rotate + `ff_ver:` (`plane.payment.flags.cache`) and the next request refetches. Observed + while creating a workspace from a Django shell; a plan/licence problem looks identical + from the outside, so check this first. - A workspace licence is **not** required for local runs: the mock flag server enables every flag regardless, and an unlicensed workspace seeds all fixtures (verified by canary against a workspace with no licence row). diff --git a/evals/env.sh b/evals/env.sh deleted file mode 100755 index 1a895c55..00000000 --- a/evals/env.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env bash -# Local eval environment bootstrap: plane-ee API + mock feature-flag server. -# -# Required env (no defaults): -# PLANE_EE_API_DIR — path to plane-ee apps/api (or monorepo root with apps/api) -# PLANE_EE_VENV — path to the Python venv used to run plane-ee manage.py -# -# Mock flags are launched via the *repo* venv (REPO/.venv/bin/python -m evals.mock_flags), -# not PLANE_EE_VENV — that venv only runs plane-ee. Both FEATURE_FLAG_SERVER_BASE_URL -# and health checks use http://127.0.0.1:9911 (mock binds 127.0.0.1). -# -# Usage: -# evals/env.sh up # start API :8000 + mock flags :9911; write evals/.env-pids -# evals/env.sh down # kill PIDs from evals/.env-pids (identity-checked) -# evals/env.sh status # report liveness -# -# API is launched with --noreload, API_KEY_RATE_LIMIT=5000/min, and -# FEATURE_FLAG_SERVER_BASE_URL=http://127.0.0.1:9911. Sources $PLANE_EE_API_DIR/.env -# (or apps/api/.env) with set -a. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -PID_FILE="$SCRIPT_DIR/.env-pids" -API_PORT=8000 -FLAG_PORT=9911 -FLAG_URL="http://127.0.0.1:${FLAG_PORT}" -API_URL="http://127.0.0.1:${API_PORT}" -FLAG_BASE_URL="http://127.0.0.1:${FLAG_PORT}" -# Repo venv used for evals.mock_flags (parameterized relative to this script). -MOCK_FLAGS_PYTHON="${REPO_ROOT}/.venv/bin/python" - -die() { echo "error: $*" >&2; exit 1; } - -require_env() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - die "$name is required (no default)" - fi -} - -resolve_api_dir() { - require_env PLANE_EE_API_DIR - require_env PLANE_EE_VENV - local base="${PLANE_EE_API_DIR}" - if [[ -d "$base/apps/api" ]]; then - echo "$base/apps/api" - elif [[ -d "$base" ]]; then - echo "$base" - else - die "PLANE_EE_API_DIR not a directory: $base" - fi -} - -resolve_python() { - local venv="${PLANE_EE_VENV}" - if [[ -x "$venv/bin/python" ]]; then - echo "$venv/bin/python" - elif [[ -x "$venv" ]]; then - echo "$venv" - else - die "PLANE_EE_VENV has no bin/python: $venv" - fi -} - -# True if $1 is a live pid whose command line looks like our managed process. -pid_is_ours() { - local pid="$1" - local kind="$2" # flag | api - if ! kill -0 "$pid" 2>/dev/null; then - return 1 - fi - local cmd - cmd="$(ps -o command= -p "$pid" 2>/dev/null || true)" - if [[ -z "$cmd" ]]; then - return 1 - fi - case "$kind" in - flag) - [[ "$cmd" == *evals.mock_flags* ]] || [[ "$cmd" == *mock_flags* ]] - ;; - api) - [[ "$cmd" == *manage.py*runserver* ]] || [[ "$cmd" == *"manage.py"*"runserver"* ]] - ;; - *) - return 1 - ;; - esac -} - -cmd_up() { - local api_dir py - api_dir="$(resolve_api_dir)" - py="$(resolve_python)" - - if [[ ! -x "$MOCK_FLAGS_PYTHON" ]]; then - die "mock-flags python not found/executable: $MOCK_FLAGS_PYTHON (create repo .venv)" - fi - - if [[ -f "$PID_FILE" ]]; then - # shellcheck disable=SC1090 - source "$PID_FILE" - local live=0 - if [[ -n "${flag_pid:-}" ]] && kill -0 "$flag_pid" 2>/dev/null; then - live=1 - fi - if [[ -n "${api_pid:-}" ]] && kill -0 "$api_pid" 2>/dev/null; then - live=1 - fi - if [[ $live -eq 1 ]]; then - die "already up (run 'down' first) — pidfile $PID_FILE has live process(es)" - fi - echo "warning: stale pidfile $PID_FILE (no live pids); replacing" >&2 - rm -f "$PID_FILE" - fi - - # Mock flag server first (API may call it on boot). - local flag_log="$SCRIPT_DIR/.mock_flags.log" - ( - cd "$REPO_ROOT" - export PLANE_EE_API_DIR - exec "$MOCK_FLAGS_PYTHON" -m evals.mock_flags "$FLAG_PORT" - ) >"$flag_log" 2>&1 & - local flag_pid=$! - echo "started mock_flags pid=$flag_pid log=$flag_log" - # Record flag_pid immediately so a later failure does not strand :9911. - { - echo "flag_pid=$flag_pid" - echo "flag_port=$FLAG_PORT" - } >"$PID_FILE" - - # Source plane-ee .env - local env_file="" - if [[ -f "$api_dir/.env" ]]; then - env_file="$api_dir/.env" - elif [[ -f "$PLANE_EE_API_DIR/.env" ]]; then - env_file="$PLANE_EE_API_DIR/.env" - fi - if [[ -n "$env_file" ]]; then - set -a - # shellcheck disable=SC1090 - source "$env_file" - set +a - echo "sourced $env_file" - else - echo "warning: no .env found under $api_dir or $PLANE_EE_API_DIR" >&2 - fi - - local api_log="$SCRIPT_DIR/.api_runserver.log" - ( - cd "$api_dir" - export API_KEY_RATE_LIMIT="5000/min" - export FEATURE_FLAG_SERVER_BASE_URL="${FLAG_BASE_URL}" - # --noreload: $! must be the real server, not the autoreloader parent. - exec "$py" manage.py runserver --noreload "0.0.0.0:${API_PORT}" - ) >"$api_log" 2>&1 & - local api_pid=$! - echo "started api runserver pid=$api_pid log=$api_log" - { - echo "flag_pid=$flag_pid" - echo "api_pid=$api_pid" - echo "flag_port=$FLAG_PORT" - echo "api_port=$API_PORT" - } >"$PID_FILE" - - # Health checks (retry briefly) - local i - for i in 1 2 3 4 5 6 7 8 9 10; do - if curl -sf -o /dev/null -X POST "$FLAG_URL/api/feature-flags/" \ - -H 'Content-Type: application/json' -d '{}'; then - echo "health: mock flags :$FLAG_PORT OK" - break - fi - if [[ $i -eq 10 ]]; then - die "mock flags health check failed on :$FLAG_PORT (see $flag_log)" - fi - sleep 0.5 - done - - for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do - if curl -sf -o /dev/null "$API_URL/" || curl -sf -o /dev/null "$API_URL/api/"; then - echo "health: api :$API_PORT OK" - break - fi - # Accept any HTTP response as "up" (auth redirects still mean listening). - if curl -s -o /dev/null -w "%{http_code}" "$API_URL/" | grep -qE '^[2345]'; then - echo "health: api :$API_PORT listening" - break - fi - if [[ $i -eq 30 ]]; then - die "api health check failed on :$API_PORT (see $api_log)" - fi - sleep 1 - done - - echo "env up: pids in $PID_FILE" -} - -cmd_down() { - if [[ ! -f "$PID_FILE" ]]; then - echo "no $PID_FILE — nothing to stop" - return 0 - fi - # shellcheck disable=SC1090 - source "$PID_FILE" - for name_kind in flag_pid:flag api_pid:api; do - local name="${name_kind%%:*}" - local kind="${name_kind##*:}" - local pid="${!name:-}" - if [[ -z "$pid" ]]; then - echo "$name=unset" - continue - fi - if ! kill -0 "$pid" 2>/dev/null; then - echo "$name=$pid not running" - continue - fi - if ! pid_is_ours "$pid" "$kind"; then - echo "warning: refusing to kill $name=$pid — command line is not a managed evals process" >&2 - ps -o command= -p "$pid" 2>/dev/null | sed 's/^/ cmd: /' >&2 || true - continue - fi - kill "$pid" 2>/dev/null || true - sleep 0.3 - if kill -0 "$pid" 2>/dev/null; then - kill -9 "$pid" 2>/dev/null || true - fi - echo "stopped $name=$pid" - done - rm -f "$PID_FILE" - echo "env down" -} - -cmd_status() { - local flag_ok=0 api_ok=0 - if curl -sf -o /dev/null -X POST "$FLAG_URL/api/feature-flags/" \ - -H 'Content-Type: application/json' -d '{}' 2>/dev/null; then - flag_ok=1 - fi - local code - code="$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/" 2>/dev/null || echo 000)" - if [[ "$code" =~ ^[2345] ]]; then - api_ok=1 - fi - echo "mock_flags :$FLAG_PORT $([[ $flag_ok -eq 1 ]] && echo UP || echo DOWN)" - echo "api :$API_PORT $([[ $api_ok -eq 1 ]] && echo UP || echo DOWN) (http $code)" - if [[ -f "$PID_FILE" ]]; then - echo "pid file: $PID_FILE" - cat "$PID_FILE" - else - echo "pid file: (none)" - fi - [[ $flag_ok -eq 1 && $api_ok -eq 1 ]] -} - -usage() { - cat < {"values": {: true}} - -Requires PLANE_EE_API_DIR (path to plane-ee apps/api, or the monorepo root -containing apps/api/plane/payment/flags/flag.py). -""" - -from __future__ import annotations - -import importlib.util -import json -import os -import sys -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - - -def _resolve_flag_module_path() -> Path: - base = os.environ.get("PLANE_EE_API_DIR", "").strip() - if not base: - raise SystemExit( - "error: PLANE_EE_API_DIR is required (path to plane-ee apps/api, or monorepo root with apps/api/...)" - ) - root = Path(base).expanduser().resolve() - candidates = [ - root / "plane" / "payment" / "flags" / "flag.py", - root / "apps" / "api" / "plane" / "payment" / "flags" / "flag.py", - ] - for c in candidates: - if c.is_file(): - return c - raise SystemExit( - f"error: FeatureFlag module not found under PLANE_EE_API_DIR={root}; " - f"tried: {', '.join(str(c) for c in candidates)}" - ) - - -def load_feature_flag_values() -> dict[str, bool]: - """Load FeatureFlag enum values and return {value: True} for every flag.""" - flag_path = _resolve_flag_module_path() - # Put the API package root on sys.path so relative imports inside flag.py work - # when the module itself only needs the enum. - api_root = flag_path.parents[3] # .../plane - package_root = flag_path.parents[4] # .../apps/api or similar - for p in (str(package_root), str(api_root.parent)): - if p not in sys.path: - sys.path.insert(0, p) - - spec = importlib.util.spec_from_file_location("plane_eval_flag", flag_path) - if spec is None or spec.loader is None: - raise SystemExit(f"error: cannot load flag module from {flag_path}") - flag_mod = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(flag_mod) - except Exception as exc: - # Fallback: read the file and eval only the enum body if full import fails - # (Django settings). Try a minimal AST-free scrape of string values. - text = flag_path.read_text(encoding="utf-8") - values: dict[str, bool] = {} - for line in text.splitlines(): - line = line.strip() - # e.g. FOO = "FOO" or FOO = "some-flag" - if "=" in line and not line.startswith("#") and not line.startswith("class"): - _, _, rhs = line.partition("=") - rhs = rhs.strip().rstrip(",") - if len(rhs) >= 2 and rhs[0] in "\"'" and rhs[-1] == rhs[0]: - values[rhs[1:-1]] = True - if values: - print( - f"mock flag server: loaded {len(values)} flags via scrape (import failed: {exc})", - flush=True, - ) - return values - raise SystemExit(f"error: failed to import FeatureFlag from {flag_path}: {exc}") from exc - - FeatureFlag = getattr(flag_mod, "FeatureFlag", None) - if FeatureFlag is None: - raise SystemExit(f"error: FeatureFlag not found in {flag_path}") - return {f.value: True for f in FeatureFlag} - - -def main(argv: list[str] | None = None) -> int: - host = "127.0.0.1" - port = 9911 - if argv is None: - argv = sys.argv[1:] - if len(argv) >= 1: - port = int(argv[0]) - - values = load_feature_flag_values() - print(f"mock flag server: {len(values)} flags all-on on {host}:{port}", flush=True) - - class Handler(BaseHTTPRequestHandler): - def do_POST(self) -> None: # noqa: N802 - body = json.dumps({"values": values}).encode() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def do_GET(self) -> None: # noqa: N802 - body = json.dumps({"values": values, "ok": True}).encode() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, *_args: object) -> None: - pass - - # Threaded: Django dev server is multi-threaded and fans out flag lookups. - ThreadingHTTPServer((host, port), Handler).serve_forever() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 128a877031a5a2df26773056305a19604f68b7d6 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 17:47:55 +0530 Subject: [PATCH 19/93] Put the two drivers at the top of the drivers package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals/drivers buried the two actual drivers a level down under filenames that described neither — api/driver.py held ApiDriver, cli/template.py held the CliDriver template — while the folders named after them held the per-vendor modules. Both drivers now live in drivers/driver.py, which leaves api/ and cli/ holding exactly what their names say. The two classes are concatenated, not refactored: they share nothing but the shape the runner calls. drivers/protocol.py is gone with them. It held an AgentDriver Protocol that nothing inherited, no type checker verified (there is none in this repo), and whose only consumer annotated its own parameter as Any — plus REPO_ROOT, which is not a protocol and was the third copy of that constant. get_driver now returns ApiDriver | CliDriver, which is both true and checkable, and REPO_ROOT is defined once in evals/__init__.py. evals/proxy.py keeps its own copy with a comment: it runs with the repo scrubbed off PYTHONPATH and cannot import its own package. Result types move to where results are defined. agent_run_to_task_result, agent_run_to_harness_dict, and AgentRun leave the driver package for evals/results.py — AgentRun has to move with them or results.py cannot import it without a cycle. token_counting.py and the MCP tool-name readers follow the same rule and graduate to evals/ now that two packages use them; the latter as tool_names.py, which is also what let results.py drop a deferred import that existed only to dodge that cycle. Restoring the two MCP translation helpers during the merge exposed that neither was tested, which is why a dropped `import json` broke no test while leaving any image-returning tool a NameError at run time. Both are covered now, and the mixed-content test was confirmed to fail without that import. --- evals/DESIGN.md | 8 +- evals/__init__.py | 8 + evals/drivers/__init__.py | 72 ++--- evals/drivers/api/__init__.py | 2 - evals/drivers/cli/__init__.py | 65 +---- evals/drivers/cli/antigravity.py | 2 +- evals/drivers/cli/claude.py | 4 +- evals/drivers/cli/codex.py | 4 +- evals/drivers/cli/opencode.py | 2 +- evals/drivers/cli/sidecar.py | 2 +- evals/drivers/cli/template.py | 278 -------------------- evals/drivers/{api => }/driver.py | 276 +++++++++++++++++++- evals/drivers/protocol.py | 363 -------------------------- evals/listing.py | 3 - evals/proxy.py | 5 +- evals/results.py | 247 +++++++++++++++++- evals/runner/live.py | 8 +- evals/{drivers => }/token_counting.py | 0 evals/tool_names.py | 102 ++++++++ tests/test_evals_api_driver.py | 64 ++++- tests/test_evals_drivers.py | 15 +- tests/test_evals_hardening.py | 6 +- tests/test_evals_proxy.py | 12 +- 23 files changed, 751 insertions(+), 797 deletions(-) delete mode 100644 evals/drivers/cli/template.py rename evals/drivers/{api => }/driver.py (61%) delete mode 100644 evals/drivers/protocol.py rename evals/{drivers => }/token_counting.py (100%) create mode 100644 evals/tool_names.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index eb27b6b8..57861470 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -209,21 +209,21 @@ evals/ debias.py I1-I5 and L1-L5 tasks and verifiers drivers/ __init__.py public exports and driver registry - protocol.py AgentDriver, AgentRun, normalization, and common row mapping - token_counting.py tool-result token sizing + driver.py AgentDriver seam, the API loop, and the CLI template api/ backend.py neutral backend protocol and turn/tool dataclasses - driver.py provider-neutral MCP/model loop anthropic.py Anthropic Messages translation openai.py OpenAI Chat Completions translation cli/ - template.py shared CLI driver template process.py subprocess lifecycle sidecar.py recording-proxy command and sidecar handling claude.py Claude Code CLI driver codex.py Codex CLI driver antigravity.py Antigravity CLI driver opencode.py OpenCode CLI driver + results.py run/task result types and common row mapping + tool_names.py whose MCP tool a call is, and what to call it + token_counting.py tool-result token sizing proxy.py stdlib-only JSON-RPC recording relay seed/ Plane fixture creation and teardown report/ summaries, A/B comparison, and multi-surface tables diff --git a/evals/__init__.py b/evals/__init__.py index fb5f7791..4347f13a 100644 --- a/evals/__init__.py +++ b/evals/__init__.py @@ -1 +1,9 @@ """Plane MCP tool-surface eval harness.""" + +from pathlib import Path + +# Repository root: the harness launches this repo's MCP server and resolves +# task working directories against it. +REPO_ROOT = Path(__file__).resolve().parent.parent + +__all__ = ["REPO_ROOT"] diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 97ede1fc..d2e01e48 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -29,57 +29,50 @@ ``response_item`` / ``function_call`` payloads (name + arguments JSON string) - Marked **experimental**; live runs are opt-in (metered quota). -This package splits that surface into focused modules (protocol types, subprocess -lifecycle, recording-proxy glue, and per-vendor drivers). Import from -``evals.drivers`` as before — public names are re-exported here. +This package splits that surface into focused modules (driver protocol, subprocess +lifecycle, recording-proxy glue, and per-vendor drivers). Driver names and their +public parse/configuration helpers are re-exported here. """ from __future__ import annotations from typing import Any -from evals.drivers.api import ApiDriver -from evals.drivers.cli import ( +from evals.drivers.cli.antigravity import ( AntigravityCliDriver, + prepare_antigravity_fake_home, + write_antigravity_mcp_config, +) +from evals.drivers.cli.claude import ( ClaudeCliDriver, - CliDriver, - CodexCliDriver, - OpencodeCliDriver, - apply_proxy_sidecar, - ensure_proxy_pythonpath, find_claude_transcript, - find_codex_rollout, - harvest_proxy_after_cli_timeout, - kill_process_group, - load_proxy_sidecar, - load_proxy_sidecar_calls, normalize_claude_usage, - note_timeout_kill, parse_claude_json_result, parse_claude_transcript_calls, + write_claude_mcp_config, +) +from evals.drivers.cli.codex import ( + CodexCliDriver, + find_codex_rollout, parse_codex_jsonl_events, parse_codex_rollout_calls, - prepare_antigravity_fake_home, - proxy_wrap_server_command, - run_cli_subprocess, - wait_for_proxy_meta, - write_antigravity_mcp_config, - write_claude_mcp_config, write_codex_mcp_override_args, +) +from evals.drivers.cli.opencode import ( + OpencodeCliDriver, write_opencode_mcp_config, ) -from evals.drivers.protocol import ( - REPO_ROOT, - AgentDriver, - AgentRun, - agent_run_to_harness_dict, - agent_run_to_task_result, - is_plane_mcp_tool, - normalize_tool_call, - split_plane_and_client_calls, - strip_mcp_prefix, +from evals.drivers.cli.process import kill_process_group, note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_wrap_server_command, + wait_for_proxy_meta, ) -from evals.results import CallRecord, TaskResult +from evals.drivers.driver import ApiDriver, CliDriver # Registry # --------------------------------------------------------------------------- @@ -87,7 +80,7 @@ KNOWN_DRIVERS = frozenset({"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) -def get_driver(name: str, **kwargs: Any) -> AgentDriver: +def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: """Return a driver instance.""" key = (name or "api").strip().lower() if key == "api": @@ -105,31 +98,22 @@ def get_driver(name: str, **kwargs: Any) -> AgentDriver: __all__ = [ "KNOWN_DRIVERS", - "AgentDriver", - "AgentRun", "AntigravityCliDriver", "ApiDriver", "ClaudeCliDriver", - "CallRecord", "CliDriver", "CodexCliDriver", "OpencodeCliDriver", - "REPO_ROOT", - "TaskResult", - "agent_run_to_harness_dict", - "agent_run_to_task_result", "apply_proxy_sidecar", "ensure_proxy_pythonpath", "find_claude_transcript", "find_codex_rollout", "get_driver", "harvest_proxy_after_cli_timeout", - "is_plane_mcp_tool", "kill_process_group", "load_proxy_sidecar", "load_proxy_sidecar_calls", "normalize_claude_usage", - "normalize_tool_call", "note_timeout_kill", "parse_claude_json_result", "parse_claude_transcript_calls", @@ -138,8 +122,6 @@ def get_driver(name: str, **kwargs: Any) -> AgentDriver: "prepare_antigravity_fake_home", "proxy_wrap_server_command", "run_cli_subprocess", - "split_plane_and_client_calls", - "strip_mcp_prefix", "wait_for_proxy_meta", "write_antigravity_mcp_config", "write_claude_mcp_config", diff --git a/evals/drivers/api/__init__.py b/evals/drivers/api/__init__.py index fef57405..1299e26a 100644 --- a/evals/drivers/api/__init__.py +++ b/evals/drivers/api/__init__.py @@ -24,7 +24,6 @@ resolve_backend_model, unregister_backend, ) -from evals.drivers.api.driver import ApiDriver from evals.drivers.api.openai import OpenAIBackend __all__ = [ @@ -32,7 +31,6 @@ "KNOWN_API_PROVIDERS", "MODEL_TIERS", "AnthropicBackend", - "ApiDriver", "BackendFactory", "BackendRegistration", "BackendRegistry", diff --git a/evals/drivers/cli/__init__.py b/evals/drivers/cli/__init__.py index 90fda0fc..44472706 100644 --- a/evals/drivers/cli/__init__.py +++ b/evals/drivers/cli/__init__.py @@ -1,64 +1 @@ -"""CLI agent drivers and their shared execution machinery.""" - -from evals.drivers.cli.antigravity import ( - AntigravityCliDriver, - prepare_antigravity_fake_home, - write_antigravity_mcp_config, -) -from evals.drivers.cli.claude import ( - ClaudeCliDriver, - find_claude_transcript, - normalize_claude_usage, - parse_claude_json_result, - parse_claude_transcript_calls, - write_claude_mcp_config, -) -from evals.drivers.cli.codex import ( - CodexCliDriver, - find_codex_rollout, - parse_codex_jsonl_events, - parse_codex_rollout_calls, - write_codex_mcp_override_args, -) -from evals.drivers.cli.opencode import OpencodeCliDriver, write_opencode_mcp_config -from evals.drivers.cli.process import kill_process_group, note_timeout_kill, run_cli_subprocess -from evals.drivers.cli.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - load_proxy_sidecar, - load_proxy_sidecar_calls, - proxy_wrap_server_command, - wait_for_proxy_meta, -) -from evals.drivers.cli.template import CliDriver - -__all__ = [ - "AntigravityCliDriver", - "ClaudeCliDriver", - "CliDriver", - "CodexCliDriver", - "OpencodeCliDriver", - "apply_proxy_sidecar", - "ensure_proxy_pythonpath", - "find_claude_transcript", - "find_codex_rollout", - "harvest_proxy_after_cli_timeout", - "kill_process_group", - "load_proxy_sidecar", - "load_proxy_sidecar_calls", - "normalize_claude_usage", - "note_timeout_kill", - "parse_claude_json_result", - "parse_claude_transcript_calls", - "parse_codex_jsonl_events", - "parse_codex_rollout_calls", - "prepare_antigravity_fake_home", - "proxy_wrap_server_command", - "run_cli_subprocess", - "wait_for_proxy_meta", - "write_antigravity_mcp_config", - "write_claude_mcp_config", - "write_codex_mcp_override_args", - "write_opencode_mcp_config", -] +"""CLI vendor drivers and their subprocess support.""" diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index bf8a786c..7530c387 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -8,7 +8,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput # Antigravity CLI (agy) — proxy-first # --------------------------------------------------------------------------- diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index c7b7472f..e4b33af9 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any -from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput, CliOutputError -from evals.drivers.protocol import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.tool_names import normalize_tool_call, split_plane_and_client_calls def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index 2b855531..add8ce6c 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -9,8 +9,8 @@ from typing import Any from evals.drivers.cli.process import run_cli_subprocess -from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput -from evals.drivers.protocol import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput +from evals.tool_names import normalize_tool_call, split_plane_and_client_calls def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py index b8402ba8..04c66aa5 100644 --- a/evals/drivers/cli/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -7,7 +7,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput # OpenCode CLI — proxy-first # --------------------------------------------------------------------------- diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 14ae6b1a..e3a862f2 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from evals.drivers.protocol import REPO_ROOT +from evals import REPO_ROOT def proxy_wrap_server_command( diff --git a/evals/drivers/cli/template.py b/evals/drivers/cli/template.py deleted file mode 100644 index 7f624d69..00000000 --- a/evals/drivers/cli/template.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Shared template for subprocess-backed CLI eval drivers.""" - -from __future__ import annotations - -import subprocess -import sys -import tempfile -import time -from abc import ABC, abstractmethod -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.cli.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - proxy_wrap_server_command, -) -from evals.drivers.protocol import REPO_ROOT, AgentRun - - -@dataclass -class CliLaunch: - """Vendor-prepared CLI launch details.""" - - cwd: Path - config_args: list[str] = field(default_factory=list) - env: dict[str, str] | None = None - - -@dataclass -class CliOutput: - """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" - - final_text: str - calls: list[dict[str, Any]] = field(default_factory=list) - client_tool_calls: list[dict[str, Any]] = field(default_factory=list) - usage: dict[str, Any] | None = None - usage_total: dict[str, Any] | None = None - stopped_reason: str = "end_turn" - raw_ref: str | None = None - call_source: str = "json" - hit_max_turns: bool = False - - -class CliOutputError(RuntimeError): - """Signal that vendor output could not produce a valid ``AgentRun``.""" - - -class CliDriver(ABC): - """Template for CLI drivers that run one MCP-backed subprocess task.""" - - name: str - experimental = False - default_call_source = "json" - run_notes: tuple[str, ...] = () - temp_dir_prefix = "plane-eval-cli-" - temp_dir_in_cwd = False - exit_note_prefix: str | None = None - include_stderr_in_exit_note = False - - def __init__( - self, - *, - python_bin: str | None = None, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - server_command: list[str] | None = None, - use_proxy: bool = True, - record_result_payloads: bool = False, - ) -> None: - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads - - def validate_run(self) -> None: - """Reject a launch before any temporary state is created, if needed.""" - return None - - @abstractmethod - def write_mcp_config( - self, - temp_dir: Path, - *, - task_cwd: Path, - server_command: list[str], - child_env: dict[str, str], - ) -> CliLaunch: - """Write vendor MCP configuration and return launch settings.""" - - @abstractmethod - def build_command( - self, - prompt: str, - *, - model: str | None, - max_turns: int, - system: str | None, - launch: CliLaunch, - ) -> list[str]: - """Build the vendor CLI command.""" - - def invoke_cli( - self, - command: list[str], - *, - launch: CliLaunch, - timeout_s: int, - ) -> subprocess.CompletedProcess[str]: - """Invoke the configured runner with the shared subprocess contract.""" - kwargs: dict[str, Any] = { - "cwd": str(launch.cwd), - "capture_output": True, - "text": True, - "timeout": timeout_s, - } - if launch.env is not None: - kwargs["env"] = launch.env - return self._runner(command, **kwargs) - - @abstractmethod - def parse_output( - self, - proc: subprocess.CompletedProcess[str], - *, - task_cwd: Path, - max_turns: int, - notes: list[str], - ) -> CliOutput: - """Parse vendor output into the normalized CLI result shape.""" - - def finalize_run( - self, - proc: subprocess.CompletedProcess[str], - *, - output: CliOutput, - notes: list[str], - ) -> None: - """Apply vendor handling that must occur after proxy reconciliation.""" - del output - if proc.returncode != 0 and self.exit_note_prefix: - notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") - stderr = proc.stderr or "" - if self.include_stderr_in_exit_note and stderr.strip(): - notes.append(stderr.strip()[:500]) - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: - """Run one CLI task using the shared configuration/proxy/timeout flow.""" - task_cwd = (cwd or REPO_ROOT).resolve() - notes = list(self.run_notes) - self.validate_run() - temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None - - with tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td: - temp_dir = Path(td) - sidecar = temp_dir / "proxy-sidecar.jsonl" - child_env = { - key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") - } - real_command = ( - list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] - ) - server_command = real_command - if self.use_proxy: - server_command = proxy_wrap_server_command( - real_command, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - ) - child_env = ensure_proxy_pythonpath(child_env) - - launch = self.write_mcp_config( - temp_dir, - task_cwd=task_cwd, - server_command=server_command, - child_env=child_env, - ) - command = self.build_command( - prompt, - model=model, - max_turns=max_turns, - system=system, - launch=launch, - ) - timeout_s = max(120, max_turns * 60) - # Persisted schema v1 defines wall time as the CLI invocation only. - started_at = time.perf_counter() - - try: - proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - started_at - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = self.default_call_source - if self.use_proxy: - calls, client_calls, call_source = harvest_proxy_after_cli_timeout( - calls, - client_calls, - sidecar, - notes, - ) - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - experimental=self.experimental, - notes=notes, - ) - - wall = time.perf_counter() - started_at - try: - output = self.parse_output( - proc, - task_cwd=task_cwd, - max_turns=max_turns, - notes=notes, - ) - except CliOutputError as exc: - if self.use_proxy: - apply_proxy_sidecar([], [], sidecar, notes) - detail = "; ".join(notes) - raise RuntimeError(f"{exc}: {detail}") from None - - if self.use_proxy: - calls, client_calls, proxy_source = apply_proxy_sidecar( - output.calls, - output.client_tool_calls, - sidecar, - notes, - ) - output.calls = calls - output.client_tool_calls = client_calls - if proxy_source == "proxy": - output.call_source = "proxy" - - self.finalize_run(proc, output=output, notes=notes) - return AgentRun( - calls=output.calls, - client_tool_calls=output.client_tool_calls, - final_text=output.final_text, - usage=output.usage, - usage_total=output.usage_total, - stopped_reason=output.stopped_reason, - raw_ref=output.raw_ref, - usage_scope="run", - call_source=output.call_source, - hit_max_turns=output.hit_max_turns, - wall_time_s=round(wall, 3), - experimental=self.experimental, - notes=notes, - ) - - -__all__ = ["CliDriver", "CliLaunch", "CliOutput", "CliOutputError"] diff --git a/evals/drivers/api/driver.py b/evals/drivers/driver.py similarity index 61% rename from evals/drivers/api/driver.py rename to evals/drivers/driver.py index af5387d9..1edeea9e 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/driver.py @@ -1,20 +1,25 @@ -"""Owned provider-neutral agent loop over a stdio MCP session.""" +"""Owned API and subprocess-backed CLI evaluation drivers.""" from __future__ import annotations import asyncio import inspect import json +import subprocess import sys +import tempfile import time +from abc import ABC, abstractmethod from collections.abc import Callable from contextlib import asynccontextmanager +from dataclasses import dataclass, field from pathlib import Path from typing import Any from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client +from evals import REPO_ROOT from evals.drivers.api.backend import ( KNOWN_API_PROVIDERS, ModelBackend, @@ -23,9 +28,15 @@ ToolSpec, create_backend, ) -from evals.drivers.protocol import AgentRun -from evals.drivers.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens -from evals.results import Usage +from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + proxy_wrap_server_command, +) +from evals.results import AgentRun, Usage +from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens DEFAULT_MAX_TOKENS = 8192 @@ -361,11 +372,268 @@ async def _run_task( ) +@dataclass +class CliLaunch: + """Vendor-prepared CLI launch details.""" + + cwd: Path + config_args: list[str] = field(default_factory=list) + env: dict[str, str] | None = None + + +@dataclass +class CliOutput: + """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" + + final_text: str + calls: list[dict[str, Any]] = field(default_factory=list) + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + usage: dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + stopped_reason: str = "end_turn" + raw_ref: str | None = None + call_source: str = "json" + hit_max_turns: bool = False + + +class CliOutputError(RuntimeError): + """Signal that vendor output could not produce a valid ``AgentRun``.""" + + +class CliDriver(ABC): + """Template for CLI drivers that run one MCP-backed subprocess task.""" + + name: str + experimental = False + default_call_source = "json" + run_notes: tuple[str, ...] = () + temp_dir_prefix = "plane-eval-cli-" + temp_dir_in_cwd = False + exit_note_prefix: str | None = None + include_stderr_in_exit_note = False + + def __init__( + self, + *, + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads + + def validate_run(self) -> None: + """Reject a launch before any temporary state is created, if needed.""" + return None + + @abstractmethod + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + """Write vendor MCP configuration and return launch settings.""" + + @abstractmethod + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + """Build the vendor CLI command.""" + + def invoke_cli( + self, + command: list[str], + *, + launch: CliLaunch, + timeout_s: int, + ) -> subprocess.CompletedProcess[str]: + """Invoke the configured runner with the shared subprocess contract.""" + kwargs: dict[str, Any] = { + "cwd": str(launch.cwd), + "capture_output": True, + "text": True, + "timeout": timeout_s, + } + if launch.env is not None: + kwargs["env"] = launch.env + return self._runner(command, **kwargs) + + @abstractmethod + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + """Parse vendor output into the normalized CLI result shape.""" + + def finalize_run( + self, + proc: subprocess.CompletedProcess[str], + *, + output: CliOutput, + notes: list[str], + ) -> None: + """Apply vendor handling that must occur after proxy reconciliation.""" + del output + if proc.returncode != 0 and self.exit_note_prefix: + notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") + stderr = proc.stderr or "" + if self.include_stderr_in_exit_note and stderr.strip(): + notes.append(stderr.strip()[:500]) + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + ) -> AgentRun: + """Run one CLI task using the shared configuration/proxy/timeout flow.""" + task_cwd = (cwd or REPO_ROOT).resolve() + notes = list(self.run_notes) + self.validate_run() + temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None + + with tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td: + temp_dir = Path(td) + sidecar = temp_dir / "proxy-sidecar.jsonl" + child_env = { + key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") + } + real_command = ( + list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] + ) + server_command = real_command + if self.use_proxy: + server_command = proxy_wrap_server_command( + real_command, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + ) + child_env = ensure_proxy_pythonpath(child_env) + + launch = self.write_mcp_config( + temp_dir, + task_cwd=task_cwd, + server_command=server_command, + child_env=child_env, + ) + command = self.build_command( + prompt, + model=model, + max_turns=max_turns, + system=system, + launch=launch, + ) + timeout_s = max(120, max_turns * 60) + # Persisted schema v1 defines wall time as the CLI invocation only. + started_at = time.perf_counter() + + try: + proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - started_at + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = self.default_call_source + if self.use_proxy: + calls, client_calls, call_source = harvest_proxy_after_cli_timeout( + calls, + client_calls, + sidecar, + notes, + ) + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + experimental=self.experimental, + notes=notes, + ) + + wall = time.perf_counter() - started_at + try: + output = self.parse_output( + proc, + task_cwd=task_cwd, + max_turns=max_turns, + notes=notes, + ) + except CliOutputError as exc: + if self.use_proxy: + apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise RuntimeError(f"{exc}: {detail}") from None + + if self.use_proxy: + calls, client_calls, proxy_source = apply_proxy_sidecar( + output.calls, + output.client_tool_calls, + sidecar, + notes, + ) + output.calls = calls + output.client_tool_calls = client_calls + if proxy_source == "proxy": + output.call_source = "proxy" + + self.finalize_run(proc, output=output, notes=notes) + return AgentRun( + calls=output.calls, + client_tool_calls=output.client_tool_calls, + final_text=output.final_text, + usage=output.usage, + usage_total=output.usage_total, + stopped_reason=output.stopped_reason, + raw_ref=output.raw_ref, + usage_scope="run", + call_source=output.call_source, + hit_max_turns=output.hit_max_turns, + wall_time_s=round(wall, 3), + experimental=self.experimental, + notes=notes, + ) + + __all__ = [ "DEFAULT_MAX_TOKENS", "KNOWN_API_PROVIDERS", "ApiDriver", "BackendFactory", + "CliDriver", + "CliLaunch", + "CliOutput", + "CliOutputError", "McpSessionFactory", "tool_result_from_mcp", "tool_spec_from_mcp", diff --git a/evals/drivers/protocol.py b/evals/drivers/protocol.py deleted file mode 100644 index 8f575a26..00000000 --- a/evals/drivers/protocol.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Shared types and tool-name helpers for eval agent drivers.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Protocol - -from evals.drivers.token_counting import ( - TOKEN_ESTIMATE_METHOD, - count_result_text_tokens, - estimate_result_tokens, -) -from evals.results import CallRecord, TaskResult, Usage - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent - -# mcp__plane__list_work_items → list_work_items -# mcp__plane-mcp-server__foo → foo -_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") -# Alternate: mcp__server__tool with multi-segment server names -_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") - - -@dataclass -class AgentRun: - """Normalized result of one agent task execution.""" - - # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} - calls: list[dict[str, Any]] - final_text: str - usage: Usage | dict[str, Any] | None - stopped_reason: str - raw_ref: str | None = None - # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics - client_tool_calls: list[dict[str, Any]] = field(default_factory=list) - # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens - usage_total: dict[str, Any] | None = None - # Harness extras (optional; defaults keep CLI paths simple) - usage_scope: str = "run" # 'run' | 'iteration' - call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'api' - hit_max_turns: bool = False - wall_time_s: float = 0.0 - experimental: bool = False - notes: list[str] = field(default_factory=list) - usage_per_iteration: list[Usage] = field(default_factory=list) - cum_input_tokens: int | None = None - result_pair_mismatch: bool = False - token_count_failures: int = 0 - # False means a tokenizer/backend counter was used for every result; True - # means at least one result used the shared character estimate. None lets - # the common row mapper determine the status from the recorded calls. - result_tokens_estimated: bool | None = None - provider: str | None = None - model: str | None = None - requested_model: str | None = None - # Raw provider finish/stop value. API drivers keep this beside the - # harness-owned normalized ``stopped_reason`` for diagnostics. - provider_stop_reason: str | None = None - - -class AgentDriver(Protocol): - """Pluggable agent backend for the evaluation harness.""" - - name: str - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - ) -> AgentRun: ... - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- - - -def strip_mcp_prefix(name: str) -> str: - """Strip Claude/Codex MCP tool name prefixes for classification. - - Examples: - mcp__plane__list_work_items → list_work_items - mcp__plane-mcp-server__find_work_items → find_work_items - """ - if not name: - return name - m = _MCP_PREFIX_RE2.match(name) - if m: - return m.group(1) - return name - - -def is_plane_mcp_tool(name: str) -> bool: - """True when the raw tool name is from our Plane MCP server (pre-strip). - - Claude surfaces MCP tools as ``mcp____``. Our config registers - the server as ``plane``, so names look like ``mcp__plane__find_work_items``. - Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. - """ - if not name: - return False - # mcp__plane__tool or mcp__plane-foo__tool - return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") - - -def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: - """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" - raw = str(name or "") - if not isinstance(args, dict): - args = {"_raw": args} - if is_plane_mcp_tool(raw): - return { - "tool": strip_mcp_prefix(raw), - "args": args, - "origin": "plane", - "raw_tool": raw, - } - return { - "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) - "args": args, - "origin": "client", - "raw_tool": raw, - } - - -def split_plane_and_client_calls( - calls: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Partition tagged calls into plane vs client lists. - - Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls - (API path) default to plane so existing harness behavior is unchanged. - """ - plane: list[dict[str, Any]] = [] - client: list[dict[str, Any]] = [] - for c in calls: - origin = c.get("origin") - if origin is None: - raw = str(c.get("raw_tool") or c.get("tool") or "") - if is_plane_mcp_tool(raw): - origin = "plane" - elif raw.startswith("mcp__"): - origin = "client" # other MCP server - else: - origin = "plane" # bare name → assume plane (API) - if origin == "client": - client.append(c) - else: - plane.append(c) - return plane, client - - -def agent_run_to_task_result( - run: AgentRun, - *, - optimal: set[str], - alternate: set[str], - classify: Callable[[str, set[str], set[str]], str], -) -> TaskResult: - """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. - - Only **plane** MCP tools are classified and counted in ``num_calls`` / - mispick metrics. Client built-ins (``ToolSearch``, …) go to - ``client_tool_calls`` and are excluded. - - CLI drivers never populate ``cum_input_tokens`` from bare - ``usage.input_tokens`` (that field is uncached-only under Claude Code and - misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. - """ - # Re-split in case callers passed a mixed list - plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) - client_src = list(run.client_tool_calls) + client_extra - - is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" - calls: list[CallRecord] = [] - local_token_count_failures = 0 - for c in plane_src: - tool = c.get("tool") or "" - args = c.get("args") or {} - try: - args_chars = len(json.dumps(args, default=str)) - except Exception: - args_chars = len(str(args)) - result_chars = int(c["result_chars"]) if c.get("result_chars") is not None else 0 - result_tokens = c.get("result_tokens") - estimated = c.get("result_tokens_estimated") - count_method = c.get("result_token_count_method") - if result_tokens is not None: - result_tokens = int(result_tokens) - if estimated is None: - estimated = bool(run.result_tokens_estimated) - if count_method is None: - count_method = TOKEN_ESTIMATE_METHOD if estimated else "backend" - elif isinstance(c.get("result_text"), str): - count = count_result_text_tokens(c["result_text"]) - result_tokens = count.value - estimated = count.estimated - count_method = count.method - local_token_count_failures += int(count.tokenizer_failed) - else: - result_tokens = estimate_result_tokens(result_chars) - estimated = True - count_method = TOKEN_ESTIMATE_METHOD - - rec = CallRecord( - tool=str(tool), - classification=classify(str(tool), optimal, alternate), - args_chars=args_chars, - result_tokens=result_tokens, - result_chars=result_chars, - result_kind=str(c.get("result_kind") or "text"), - is_error=bool(c.get("is_error")), - result_tokens_estimated=bool(estimated), - result_token_count_method=str(count_method), - duration_ms=c.get("duration_ms"), - ) - # Action-dispatch surfaces: the action arg IS the second half of the - # tool choice — keep it (args content is otherwise not persisted). - if isinstance(args, dict) and isinstance(args.get("action"), str): - rec.action = args["action"] - calls.append(rec) - - client_tool_calls: list[CallRecord] = [] - for c in client_src: - tool = c.get("tool") or c.get("raw_tool") or "" - args = c.get("args") or {} - try: - args_chars = len(json.dumps(args, default=str)) - except Exception: - args_chars = len(str(args)) - client_tool_calls.append( - CallRecord( - tool=str(tool), - args_chars=args_chars, - raw_tool=str(c.get("raw_tool") or tool), - ) - ) - - stop_reason = run.stopped_reason - hit_max = run.hit_max_turns - if hit_max: - stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" - - errored = sum(1 for c in calls if c.is_error) - alternate_n = sum(1 for c in calls if c.classification == "alternate") - out_of_set_n = sum(1 for c in calls if c.classification == "out_of_set") - - # CLI path: never write misleading cum_input_tokens from uncached-only field. - # usage_total is driver-owned — do not re-derive it here (Claude vs Codex - # shapes differ; a generic Claude rebuild mislabels other vendors). - usage_total = run.usage_total - - if run.usage_per_iteration: - usage_per_iteration = list(run.usage_per_iteration) - cum_input = ( - run.cum_input_tokens - if run.cum_input_tokens is not None - else sum(item.input_tokens for item in usage_per_iteration) - ) - cum_reason = None - elif is_cli: - cum_input: int | None = None - cum_reason: str | None = ( - "CLI driver: Claude usage.input_tokens is uncached-only; " - "see usage_total (cache_read/cache_creation/output/cost) for run accounting" - ) - usage_per_iteration: list[Usage] = [] - else: - cum_input = 0 - cum_reason = None - usage_per_iteration = [] - if run.usage and run.usage_scope == "iteration": - pass - - estimated_states = [bool(c.result_tokens_estimated) for c in calls] - if estimated_states: - result_tokens_estimated = any(estimated_states) - result_tokens_mode = ( - "estimated" if all(estimated_states) else "measured" if not any(estimated_states) else "mixed" - ) - else: - result_tokens_estimated = ( - bool(run.result_tokens_estimated) if run.result_tokens_estimated is not None else is_cli - ) - result_tokens_mode = "estimated" if result_tokens_estimated else "measured" - - count_methods = {str(c.result_token_count_method) for c in calls} - if not count_methods: - result_token_count_method = "none" - elif len(count_methods) == 1: - result_token_count_method = next(iter(count_methods)) - else: - result_token_count_method = "mixed" - return TaskResult( - final_text=run.final_text, - calls=calls, - num_calls=len(calls), - client_tool_calls=client_tool_calls, - client_tool_call_count=len(client_tool_calls), - errored_calls=errored, - alternate_calls=alternate_n, - out_of_set_calls=out_of_set_n, - total_result_tokens=sum(int(c.result_tokens or 0) for c in calls), - usage_per_iteration=usage_per_iteration, - cum_input_tokens=cum_input, - cum_input_tokens_reason=cum_reason, - wall_time_s=run.wall_time_s, - stop_reason=stop_reason, - provider_stop_reason=run.provider_stop_reason, - hit_max_iterations=hit_max, - result_pair_mismatch=run.result_pair_mismatch, - token_count_failures=run.token_count_failures + local_token_count_failures, - result_tokens_estimated=result_tokens_estimated, - result_tokens_mode=result_tokens_mode, - result_token_count_method=result_token_count_method, - usage_scope=run.usage_scope, - call_source=run.call_source, - driver_raw_ref=run.raw_ref, - driver_notes=list(run.notes), - usage=run.usage, - usage_total=usage_total, - provider=run.provider, - model=run.model, - requested_model=run.requested_model, - ) - - -def agent_run_to_harness_dict( - run: AgentRun, - *, - optimal: set[str], - alternate: set[str], - classify: Callable[[str, set[str], set[str]], str], -) -> dict[str, Any]: - """Compatibility wrapper returning the public persisted-row dictionary.""" - return agent_run_to_task_result( - run, - optimal=optimal, - alternate=alternate, - classify=classify, - ).to_row() - - -__all__ = [ - "REPO_ROOT", - "AgentRun", - "AgentDriver", - "agent_run_to_harness_dict", - "agent_run_to_task_result", - "is_plane_mcp_tool", - "normalize_tool_call", - "split_plane_and_client_calls", - "strip_mcp_prefix", -] diff --git a/evals/listing.py b/evals/listing.py index 7eb06f97..4fedc1de 100644 --- a/evals/listing.py +++ b/evals/listing.py @@ -20,13 +20,10 @@ import shlex import sys from dataclasses import dataclass -from pathlib import Path from typing import Any from evals.runner.live import stdio_server_env -REPO_ROOT = Path(__file__).resolve().parent.parent - @dataclass class ToolTokenRow: diff --git a/evals/proxy.py b/evals/proxy.py index df73fb14..22c9b9c4 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -33,7 +33,10 @@ SHUTDOWN_DEADLINE_S = 10.0 READ_CHUNK = 65536 -# Repo root for PYTHONPATH scrubbing (parent of evals/). +# Repo root for PYTHONPATH scrubbing (parent of evals/). Deliberately computed +# here rather than imported from ``evals``: this module runs inside the MCP +# server's process tree with the repo scrubbed off PYTHONPATH, so it cannot +# import its own package. REPO_ROOT = Path(__file__).resolve().parent.parent diff --git a/evals/results.py b/evals/results.py index 8f493eb4..33ae7e1c 100644 --- a/evals/results.py +++ b/evals/results.py @@ -2,9 +2,18 @@ from __future__ import annotations +import json +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any, Literal +from evals.token_counting import ( + TOKEN_ESTIMATE_METHOD, + count_result_text_tokens, + estimate_result_tokens, +) +from evals.tool_names import split_plane_and_client_calls + RESULT_SCHEMA_VERSION = 1 @@ -37,6 +46,43 @@ class CallRecord: result_tokens_skipped: str | None = None +@dataclass +class AgentRun: + """Normalized result of one agent task execution.""" + + # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} + calls: list[dict[str, Any]] + final_text: str + usage: Usage | dict[str, Any] | None + stopped_reason: str + raw_ref: str | None = None + # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens + usage_total: dict[str, Any] | None = None + # Harness extras (optional; defaults keep CLI paths simple) + usage_scope: str = "run" # 'run' | 'iteration' + call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'api' + hit_max_turns: bool = False + wall_time_s: float = 0.0 + experimental: bool = False + notes: list[str] = field(default_factory=list) + usage_per_iteration: list[Usage] = field(default_factory=list) + cum_input_tokens: int | None = None + result_pair_mismatch: bool = False + token_count_failures: int = 0 + # False means a tokenizer/backend counter was used for every result; True + # means at least one result used the shared character estimate. None lets + # the common row mapper determine the status from the recorded calls. + result_tokens_estimated: bool | None = None + provider: str | None = None + model: str | None = None + requested_model: str | None = None + # Raw provider finish/stop value. API drivers keep this beside the + # harness-owned normalized ``stopped_reason`` for diagnostics. + provider_stop_reason: str | None = None + + @dataclass(slots=True) class TaskResult: """One task repetition and the complete persisted row schema. @@ -391,4 +437,203 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ) -__all__ = ["RESULT_SCHEMA_VERSION", "CallRecord", "TaskResult", "Usage"] +def agent_run_to_task_result( + run: AgentRun, + *, + optimal: set[str], + alternate: set[str], + classify: Callable[[str, set[str], set[str]], str], +) -> TaskResult: + """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. + + Only **plane** MCP tools are classified and counted in ``num_calls`` / + mispick metrics. Client built-ins (``ToolSearch``, …) go to + ``client_tool_calls`` and are excluded. + + CLI drivers never populate ``cum_input_tokens`` from bare + ``usage.input_tokens`` (that field is uncached-only under Claude Code and + misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. + """ + # Re-split in case callers passed a mixed list + plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) + client_src = list(run.client_tool_calls) + client_extra + + is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" + calls: list[CallRecord] = [] + local_token_count_failures = 0 + for c in plane_src: + tool = c.get("tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + result_chars = int(c["result_chars"]) if c.get("result_chars") is not None else 0 + result_tokens = c.get("result_tokens") + estimated = c.get("result_tokens_estimated") + count_method = c.get("result_token_count_method") + if result_tokens is not None: + result_tokens = int(result_tokens) + if estimated is None: + estimated = bool(run.result_tokens_estimated) + if count_method is None: + count_method = TOKEN_ESTIMATE_METHOD if estimated else "backend" + elif isinstance(c.get("result_text"), str): + count = count_result_text_tokens(c["result_text"]) + result_tokens = count.value + estimated = count.estimated + count_method = count.method + local_token_count_failures += int(count.tokenizer_failed) + else: + result_tokens = estimate_result_tokens(result_chars) + estimated = True + count_method = TOKEN_ESTIMATE_METHOD + + rec = CallRecord( + tool=str(tool), + classification=classify(str(tool), optimal, alternate), + args_chars=args_chars, + result_tokens=result_tokens, + result_chars=result_chars, + result_kind=str(c.get("result_kind") or "text"), + is_error=bool(c.get("is_error")), + result_tokens_estimated=bool(estimated), + result_token_count_method=str(count_method), + duration_ms=c.get("duration_ms"), + ) + # Action-dispatch surfaces: the action arg IS the second half of the + # tool choice — keep it (args content is otherwise not persisted). + if isinstance(args, dict) and isinstance(args.get("action"), str): + rec.action = args["action"] + calls.append(rec) + + client_tool_calls: list[CallRecord] = [] + for c in client_src: + tool = c.get("tool") or c.get("raw_tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + client_tool_calls.append( + CallRecord( + tool=str(tool), + args_chars=args_chars, + raw_tool=str(c.get("raw_tool") or tool), + ) + ) + + stop_reason = run.stopped_reason + hit_max = run.hit_max_turns + if hit_max: + stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" + + errored = sum(1 for c in calls if c.is_error) + alternate_n = sum(1 for c in calls if c.classification == "alternate") + out_of_set_n = sum(1 for c in calls if c.classification == "out_of_set") + + # CLI path: never write misleading cum_input_tokens from uncached-only field. + # usage_total is driver-owned — do not re-derive it here (Claude vs Codex + # shapes differ; a generic Claude rebuild mislabels other vendors). + usage_total = run.usage_total + + if run.usage_per_iteration: + usage_per_iteration = list(run.usage_per_iteration) + cum_input = ( + run.cum_input_tokens + if run.cum_input_tokens is not None + else sum(item.input_tokens for item in usage_per_iteration) + ) + cum_reason = None + elif is_cli: + cum_input: int | None = None + cum_reason: str | None = ( + "CLI driver: Claude usage.input_tokens is uncached-only; " + "see usage_total (cache_read/cache_creation/output/cost) for run accounting" + ) + usage_per_iteration: list[Usage] = [] + else: + cum_input = 0 + cum_reason = None + usage_per_iteration = [] + if run.usage and run.usage_scope == "iteration": + pass + + estimated_states = [bool(c.result_tokens_estimated) for c in calls] + if estimated_states: + result_tokens_estimated = any(estimated_states) + result_tokens_mode = ( + "estimated" if all(estimated_states) else "measured" if not any(estimated_states) else "mixed" + ) + else: + result_tokens_estimated = ( + bool(run.result_tokens_estimated) if run.result_tokens_estimated is not None else is_cli + ) + result_tokens_mode = "estimated" if result_tokens_estimated else "measured" + + count_methods = {str(c.result_token_count_method) for c in calls} + if not count_methods: + result_token_count_method = "none" + elif len(count_methods) == 1: + result_token_count_method = next(iter(count_methods)) + else: + result_token_count_method = "mixed" + return TaskResult( + final_text=run.final_text, + calls=calls, + num_calls=len(calls), + client_tool_calls=client_tool_calls, + client_tool_call_count=len(client_tool_calls), + errored_calls=errored, + alternate_calls=alternate_n, + out_of_set_calls=out_of_set_n, + total_result_tokens=sum(int(c.result_tokens or 0) for c in calls), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=cum_input, + cum_input_tokens_reason=cum_reason, + wall_time_s=run.wall_time_s, + stop_reason=stop_reason, + provider_stop_reason=run.provider_stop_reason, + hit_max_iterations=hit_max, + result_pair_mismatch=run.result_pair_mismatch, + token_count_failures=run.token_count_failures + local_token_count_failures, + result_tokens_estimated=result_tokens_estimated, + result_tokens_mode=result_tokens_mode, + result_token_count_method=result_token_count_method, + usage_scope=run.usage_scope, + call_source=run.call_source, + driver_raw_ref=run.raw_ref, + driver_notes=list(run.notes), + usage=run.usage, + usage_total=usage_total, + provider=run.provider, + model=run.model, + requested_model=run.requested_model, + ) + + +def agent_run_to_harness_dict( + run: AgentRun, + *, + optimal: set[str], + alternate: set[str], + classify: Callable[[str, set[str], set[str]], str], +) -> dict[str, Any]: + """Compatibility wrapper returning the public persisted-row dictionary.""" + return agent_run_to_task_result( + run, + optimal=optimal, + alternate=alternate, + classify=classify, + ).to_row() + + +__all__ = [ + "RESULT_SCHEMA_VERSION", + "AgentRun", + "CallRecord", + "TaskResult", + "Usage", + "agent_run_to_harness_dict", + "agent_run_to_task_result", +] diff --git a/evals/runner/live.py b/evals/runner/live.py index 3c3668ba..611a31a8 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -11,9 +11,9 @@ from pathlib import Path from typing import Any -from evals.drivers import KNOWN_DRIVERS, agent_run_to_task_result, get_driver +from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS -from evals.results import TaskResult +from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown from evals.tasks import ( PromptBindError, @@ -100,7 +100,7 @@ async def run_agent_task_via_driver( alternate_tools: set[str] | None = None, server_env: dict[str, str] | None = None, ) -> TaskResult: - """Run one task through the selected AgentDriver.""" + """Run one task through the selected driver.""" project_name = ctx["project_name"] system = _system_preamble(workspace_slug, project_name) prompt = format_task_prompt(task, ctx, strict=True) @@ -109,7 +109,7 @@ async def run_agent_task_via_driver( assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" mcp_env = stdio_server_env(extra=server_env) - # AgentDriver is sync (CLI subprocess or API loop); keep it off this loop. + # Drivers are sync (CLI subprocess or API loop); keep them off this loop. agent_run = await asyncio.to_thread( driver.run_task, prompt, diff --git a/evals/drivers/token_counting.py b/evals/token_counting.py similarity index 100% rename from evals/drivers/token_counting.py rename to evals/token_counting.py diff --git a/evals/tool_names.py b/evals/tool_names.py new file mode 100644 index 00000000..cb890459 --- /dev/null +++ b/evals/tool_names.py @@ -0,0 +1,102 @@ +"""Read an MCP tool name: whose tool it is, and what to call it. + +Agent CLIs expose MCP tools under a vendor prefix (``mcp__plane__list_work_items``) +and mix them with their own built-ins (``Bash``, ``ToolSearch``). Drivers and the +result mapper both have to tell those apart before anything is classified or +counted, so this sits beside the result schema rather than inside one driver +package. +""" + +from __future__ import annotations + +import re +from typing import Any + +# mcp__plane__list_work_items → list_work_items +# mcp__plane-mcp-server__foo → foo +_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") +# Alternate: mcp__server__tool with multi-segment server names +_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") + + +def strip_mcp_prefix(name: str) -> str: + """Strip Claude/Codex MCP tool name prefixes for classification. + + Examples: + mcp__plane__list_work_items → list_work_items + mcp__plane-mcp-server__find_work_items → find_work_items + """ + if not name: + return name + m = _MCP_PREFIX_RE2.match(name) + if m: + return m.group(1) + return name + + +def is_plane_mcp_tool(name: str) -> bool: + """True when the raw tool name is from our Plane MCP server (pre-strip). + + Claude surfaces MCP tools as ``mcp____``. Our config registers + the server as ``plane``, so names look like ``mcp__plane__find_work_items``. + Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. + """ + if not name: + return False + # mcp__plane__tool or mcp__plane-foo__tool + return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") + + +def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: + """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" + raw = str(name or "") + if not isinstance(args, dict): + args = {"_raw": args} + if is_plane_mcp_tool(raw): + return { + "tool": strip_mcp_prefix(raw), + "args": args, + "origin": "plane", + "raw_tool": raw, + } + return { + "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) + "args": args, + "origin": "client", + "raw_tool": raw, + } + + +def split_plane_and_client_calls( + calls: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Partition tagged calls into plane vs client lists. + + Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls + (API path) default to plane so existing harness behavior is unchanged. + """ + plane: list[dict[str, Any]] = [] + client: list[dict[str, Any]] = [] + for c in calls: + origin = c.get("origin") + if origin is None: + raw = str(c.get("raw_tool") or c.get("tool") or "") + if is_plane_mcp_tool(raw): + origin = "plane" + elif raw.startswith("mcp__"): + origin = "client" # other MCP server + else: + origin = "plane" # bare name → assume plane (API) + if origin == "client": + client.append(c) + else: + plane.append(c) + return plane, client + + +__all__ = [ + "is_plane_mcp_tool", + "normalize_tool_call", + "split_plane_and_client_calls", + "strip_mcp_prefix", +] diff --git a/tests/test_evals_api_driver.py b/tests/test_evals_api_driver.py index 882f1077..1a411616 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/test_evals_api_driver.py @@ -10,11 +10,10 @@ import pytest -from evals.drivers import agent_run_to_harness_dict +from evals.drivers import ApiDriver from evals.drivers.api import ( KNOWN_API_PROVIDERS, AnthropicBackend, - ApiDriver, OpenAIBackend, StopReason, ToolCall, @@ -27,7 +26,8 @@ resolve_backend_model, unregister_backend, ) -from evals.drivers.token_counting import estimate_result_tokens +from evals.results import agent_run_to_harness_dict +from evals.token_counting import estimate_result_tokens class FakeBackend: @@ -678,3 +678,61 @@ def test_openai_backend_normalizes_refusal_for_driver_guard(): assert turn.provider_stop_reason == "content_filter" assert turn.text == "declined" assert turn.tool_calls == [ToolCall("danger", "write", {})] + + +# --------------------------------------------------------------------------- +# MCP translation helpers +# --------------------------------------------------------------------------- + + +def test_tool_spec_from_mcp_reads_dict_and_object_entries(): + from evals.drivers.driver import tool_spec_from_mcp + + as_dict = tool_spec_from_mcp( + {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} + ) + assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") + assert as_dict.input_schema == {"type": "object", "x": 1} + + as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) + assert as_object.name == "create_cycle" + # A missing or non-dict schema must still yield a usable object schema. + assert as_object.input_schema == {"type": "object"} + assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} + + +def test_tool_result_from_mcp_text_only_joins_blocks(): + from evals.drivers.driver import tool_result_from_mcp + + result = tool_result_from_mcp( + "call_1", + {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + ) + assert (result.call_id, result.text, result.kind, result.is_error) == ("call_1", "first\nsecond", "text", False) + + +def test_tool_result_from_mcp_serializes_non_text_blocks(): + """A tool returning an image must not be counted as if it returned nothing. + + This path serializes the whole content list, so it is the only one that + needs ``json`` at run time — a missing import here fails no other test. + """ + from evals.drivers.driver import tool_result_from_mcp + + mixed = tool_result_from_mcp( + "call_2", + {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, + ) + assert mixed.kind == "mixed" + assert '"image"' in mixed.text and "chart:" in mixed.text + + image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) + assert image_only.kind == "image" + assert '"data":"AAAA"' in image_only.text + + +def test_tool_result_from_mcp_propagates_error_flag_in_both_spellings(): + from evals.drivers.driver import tool_result_from_mcp + + assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True + assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True diff --git a/tests/test_evals_drivers.py b/tests/test_evals_drivers.py index c4db0492..030a407e 100644 --- a/tests/test_evals_drivers.py +++ b/tests/test_evals_drivers.py @@ -16,25 +16,26 @@ from evals.cli import parse_args from evals.drivers import ( KNOWN_DRIVERS, - AgentRun, ApiDriver, ClaudeCliDriver, CodexCliDriver, - agent_run_to_harness_dict, get_driver, - is_plane_mcp_tool, normalize_claude_usage, - normalize_tool_call, parse_claude_json_result, parse_claude_transcript_calls, parse_codex_jsonl_events, run_cli_subprocess, - split_plane_and_client_calls, - strip_mcp_prefix, write_claude_mcp_config, ) -from evals.drivers.token_counting import estimate_result_tokens +from evals.results import AgentRun, agent_run_to_harness_dict from evals.runner.live import classify_call, stdio_server_env +from evals.token_counting import estimate_result_tokens +from evals.tool_names import ( + is_plane_mcp_tool, + normalize_tool_call, + split_plane_and_client_calls, + strip_mcp_prefix, +) # --------------------------------------------------------------------------- # Fixtures (constructed — never captured from live CLIs) diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 8b3ae694..a2386d8a 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -15,9 +15,9 @@ from evals import cli as run_mod from evals import report as report_mod from evals import seed as seed_mod -from evals.drivers import AgentRun, ClaudeCliDriver, parse_claude_json_result +from evals.drivers import ClaudeCliDriver, parse_claude_json_result from evals.report import is_infra_error_row, load_rows, summarize -from evals.results import RESULT_SCHEMA_VERSION, TaskResult +from evals.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult from evals.runner import canary as runner_canary from evals.runner import ( is_infra_cli_stop_reason, @@ -325,7 +325,7 @@ def run_task(self, *args, **kwargs): def test_run_live_timeout_agent_is_infra_cli(tmp_path: Path, monkeypatch): """Driver returns stopped_reason=timeout → row error_class=infra_cli, battery continues.""" - from evals.drivers import agent_run_to_harness_dict + from evals.results import agent_run_to_harness_dict out = tmp_path / "rows.jsonl" fake_plane = MagicMock() diff --git a/tests/test_evals_proxy.py b/tests/test_evals_proxy.py index 7427f97a..2e2b3ef6 100644 --- a/tests/test_evals_proxy.py +++ b/tests/test_evals_proxy.py @@ -18,7 +18,6 @@ ClaudeCliDriver, CodexCliDriver, OpencodeCliDriver, - agent_run_to_harness_dict, apply_proxy_sidecar, ensure_proxy_pythonpath, get_driver, @@ -31,8 +30,7 @@ write_antigravity_mcp_config, write_opencode_mcp_config, ) -from evals.drivers.cli.template import CliDriver, CliLaunch, CliOutput -from evals.drivers.token_counting import estimate_result_tokens +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -43,6 +41,8 @@ write_all_fd, ) from evals.proxy import main as proxy_main +from evals.results import AgentRun, agent_run_to_harness_dict +from evals.token_counting import estimate_result_tokens REPO = Path(__file__).resolve().parent.parent @@ -539,8 +539,6 @@ def fake_run(cmd, **kwargs): def test_agent_run_to_harness_propagates_proxy_fields(): - from evals.drivers import AgentRun - run = AgentRun( calls=[ { @@ -580,7 +578,7 @@ def test_agent_run_to_harness_propagates_proxy_fields(): def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.cli.template.time.perf_counter", lambda: clock["now"]) + monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) class MinimalCliDriver(CliDriver): name = "minimal-cli" @@ -1842,8 +1840,6 @@ def fake_get_driver(name, **kwargs): class Dummy: def run_task(self, *a, **k): - from evals.drivers import AgentRun - return AgentRun( calls=[], final_text="", From 694ae76cac705d85664b53c9fa004cfdd874702e Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 18:16:08 +0530 Subject: [PATCH 20/93] Split the task package by concern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals/tasks/common.py held three unrelated jobs under a name that described none of them and is on our banned list: binding a prompt to a seeded fixture, grading an agent's answer against a contract, and reading Plane to establish what is actually true. Those are now prompts.py, answers.py, and lookups.py, so a verifier author can see which of the three they need. TaskSkipped gets its own module rather than sitting beside PromptBindError. A skip is not a failure anywhere in this harness — the report keeps skips out of success denominators — and filing it under errors would blur the one distinction the reporting is careful about. tasks/__init__.py is now re-exports only; the catalog assembly, the pinned id order, and the fingerprint hash live in catalog.py. The package's public face is unchanged: `from evals.tasks import TASKS, get_tasks, TaskSkipped, ...` still works, and the task modules now import the specific module they need instead of a re-export. Placement only, proven by the fingerprint: it hashes every task's prompt and tool sets, and it is byte-identical at d546d3181bdb across 34 tasks in the same pinned order. All 22 definitions moved verbatim. --- evals/DESIGN.md | 8 +- evals/README.md | 7 +- evals/cli.py | 3 +- evals/runner/canary.py | 3 +- evals/runner/live.py | 10 +- evals/seed/work_items.py | 2 +- evals/tasks/__init__.py | 151 +++---------- evals/tasks/answers.py | 113 ++++++++++ evals/tasks/catalog.py | 117 ++++++++++ evals/tasks/common.py | 307 --------------------------- evals/tasks/cross.py | 6 +- evals/tasks/debias.py | 5 +- evals/tasks/lookups.py | 133 ++++++++++++ evals/tasks/prompts.py | 67 ++++++ evals/tasks/read.py | 6 +- evals/tasks/schema.py | 3 +- evals/tasks/skip.py | 12 ++ evals/tasks/write.py | 4 +- tests/test_evals_catalog.py | 4 +- tests/test_evals_debias_verifiers.py | 29 +-- tests/test_evals_hardening.py | 7 +- tests/test_evals_output_contracts.py | 3 +- tests/test_evals_verifiers.py | 16 +- 23 files changed, 521 insertions(+), 495 deletions(-) create mode 100644 evals/tasks/answers.py create mode 100644 evals/tasks/catalog.py delete mode 100644 evals/tasks/common.py create mode 100644 evals/tasks/lookups.py create mode 100644 evals/tasks/prompts.py create mode 100644 evals/tasks/skip.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 57861470..e0c4360e 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -200,8 +200,12 @@ evals/ meta.py run metadata and repository provenance canary.py empty-agent verifier canary tasks/ - __init__.py ordered catalog assembly and public task API - common.py prompt binding, matchers, and shared API lookups + __init__.py public task API re-exports + catalog.py ordered catalog assembly and fingerprinting + prompts.py task prompt binding + answers.py answer-contract matching + lookups.py Plane reads used to establish verifier truth + skip.py task skip signal read.py R1-R7 tasks and verifiers write.py W1-W10 tasks and verifiers schema.py S1-S5 tasks and verifiers diff --git a/evals/README.md b/evals/README.md index b81e14d9..6899657c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -171,8 +171,9 @@ verifiers: - `cross.py`: C1-C2 - `debias.py`: I1-I5 and L1-L5 -Shared prompt, matcher, and API lookup machinery lives in `common.py`. `tasks/__init__.py` -assembles the class lists in the pinned catalog order and re-exports the public task API. +Prompt binding, answer matching, Plane lookups, and the skip signal live in `prompts.py`, +`answers.py`, `lookups.py`, and `skip.py`. `catalog.py` assembles the class lists in the +pinned catalog order, while `tasks/__init__.py` re-exports the public task API. A task is a dict: ```python @@ -197,7 +198,7 @@ variants (e.g. `cycles_open_past`) don't leak between tasks. Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)`. Keep a new task and its verifier in the same class module, add it to that module's exported task list, and -preserve the assembly order in `tasks/__init__.py`. +preserve the assembly order in `tasks/catalog.py`. Mutation verifiers must read the resulting state through the Plane API. Read verifiers must derive the expected facts from the API or seed context and match an explicit answer contract diff --git a/evals/cli.py b/evals/cli.py index dbfa0b8d..134efd5d 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -20,7 +20,8 @@ ) from evals.runner import run_canary, run_live from evals.seed import seed_plan -from evals.tasks import TASKS, format_task_prompt, get_tasks +from evals.tasks.catalog import TASKS, get_tasks +from evals.tasks.prompts import format_task_prompt API_MODEL_TIERS: dict[str, dict[str, str]] = { provider: aliases for provider in KNOWN_API_PROVIDERS if (aliases := backend_model_aliases(provider)) diff --git a/evals/runner/canary.py b/evals/runner/canary.py index 50f93faf..5672bdcb 100644 --- a/evals/runner/canary.py +++ b/evals/runner/canary.py @@ -7,7 +7,8 @@ from typing import Any from evals.seed import make_plane_client, seed, teardown -from evals.tasks import TaskSkipped, battery_fingerprint +from evals.tasks.catalog import battery_fingerprint +from evals.tasks.skip import TaskSkipped async def run_canary( diff --git a/evals/runner/live.py b/evals/runner/live.py index 611a31a8..243914da 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -15,13 +15,9 @@ from evals.drivers.api import MODEL_TIERS from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown -from evals.tasks import ( - PromptBindError, - TaskSkipped, - battery_fingerprint, - format_task_prompt, - task_author, -) +from evals.tasks.catalog import battery_fingerprint, task_author +from evals.tasks.prompts import PromptBindError, format_task_prompt +from evals.tasks.skip import TaskSkipped from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision from .resume import load_resume_skip_keys diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 868c9562..accf0a3a 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -148,7 +148,7 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st Raises :class:`evals.tasks.TaskSkipped` with reason ``env:no-activity-worker`` so the harness records a skip, not a task failure. """ - from evals.tasks import TaskSkipped + from evals.tasks.skip import TaskSkipped project_id = context.get("project_id") work_item_id = (context.get("items") or {}).get(CHECKOUT_TIMEOUT_TITLE) diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index c6c6fe02..7180ad0b 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -1,35 +1,25 @@ -"""Public task catalog assembled from task-class modules.""" +"""Public task catalog and verifier API.""" -from __future__ import annotations - -import hashlib -import json -from typing import Any - -from evals.tasks.common import ( - PromptBindError, - TaskSkipped, - as_id, +from evals.tasks.answers import ( contract_values, - count_open_urgent, - find_item_by_name, - find_items_by_name, - format_task_prompt, get_final_text, - ids, - is_not_found, reports_contract_int, reports_contract_value, reports_contract_values, reports_exact_int, - state_group, - state_name, whole_answer_int, word_boundary, ) -from evals.tasks.cross import CROSS_TASKS, verify_c1, verify_c2 +from evals.tasks.catalog import ( + EXPECTED_TASK_IDS, + TASKS, + TASKS_BY_ID, + battery_fingerprint, + get_tasks, + task_author, +) +from evals.tasks.cross import verify_c1, verify_c2 from evals.tasks.debias import ( - DEBIAS_TASKS, I1_TITLE, I2_TITLE, I3_TITLE, @@ -51,19 +41,21 @@ verify_l4, verify_l5, ) -from evals.tasks.read import ( - READ_TASKS, - verify_r1, - verify_r2, - verify_r3, - verify_r4, - verify_r5, - verify_r6, - verify_r7, +from evals.tasks.lookups import ( + as_id, + count_open_urgent, + find_item_by_name, + find_items_by_name, + ids, + is_not_found, + state_group, + state_name, ) -from evals.tasks.schema import SCHEMA_TASKS, verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.prompts import PromptBindError, format_task_prompt +from evals.tasks.read import verify_r1, verify_r2, verify_r3, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.skip import TaskSkipped from evals.tasks.write import ( - WRITE_TASKS, verify_w1, verify_w2, verify_w3, @@ -76,101 +68,6 @@ verify_w10, ) -EXPECTED_TASK_IDS = ( - "R1", - "R2", - "R3", - "R4", - "R5", - "R6", - "W1", - "W2", - "W3", - "W4", - "W5", - "W6", - "W7", - "W8", - "W9", - "W10", - "S1", - "S2", - "S3", - "S4", - "S5", - "C1", - "C2", - "R7", - "I1", - "I2", - "I3", - "I4", - "I5", - "L1", - "L2", - "L3", - "L4", - "L5", -) - -# Preserve the historical catalog order exactly: R7 was added after C1/C2. -TASKS: list[dict[str, Any]] = [ - *READ_TASKS[:6], - *WRITE_TASKS, - *SCHEMA_TASKS, - *CROSS_TASKS, - READ_TASKS[6], - *DEBIAS_TASKS, -] -if tuple(task["id"] for task in TASKS) != EXPECTED_TASK_IDS: - raise RuntimeError("assembled task order changed; battery/result compatibility would break") - -TASKS_BY_ID: dict[str, dict[str, Any]] = {task["id"]: task for task in TASKS} - - -def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: - """Return tasks filtered by id list (None = all).""" - if ids is None: - return list(TASKS) - missing = [i for i in ids if i not in TASKS_BY_ID] - if missing: - raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") - return [TASKS_BY_ID[i] for i in ids] - - -def task_author(task: dict[str, Any]) -> str: - """Return the task author; default ``claude`` when the key is absent.""" - return str(task.get("author") or "claude") - - -def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: - """Stable short hash of the task battery used for a run. - - SHA-256 (first 12 hex chars) over a canonical serialization of every task - sorted by id: id, prompt, sorted optimal/alternate tools, and optimal_calls. - - Ceilings (intentionally *not* covered by the hash): - - Verifier functions and ``needs`` fixtures do not alter the fingerprint — - prompt/tool-set drift is the stability signal, not seed/verify logic. - - The hash covers the *selected* task list: ``--tasks`` subsets produce - different fingerprints than a full-catalog run. - """ - src = list(TASKS if tasks is None else tasks) - payload: list[dict[str, Any]] = [] - for t in sorted(src, key=lambda x: str(x.get("id") or "")): - payload.append( - { - "id": t.get("id"), - "prompt": t.get("prompt"), - "optimal_tools": sorted(t.get("optimal_tools") or []), - "alternate_tools": sorted(t.get("alternate_tools") or []), - "optimal_calls": t.get("optimal_calls"), - } - ) - blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] - - __all__ = [ "EXPECTED_TASK_IDS", "PromptBindError", diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py new file mode 100644 index 00000000..7868af55 --- /dev/null +++ b/evals/tasks/answers.py @@ -0,0 +1,113 @@ +"""Answer-contract matching for task verifiers.""" + +from __future__ import annotations + +import re +from collections import Counter +from typing import Any + + +def word_boundary(value: str) -> re.Pattern[str]: + """Compile a case-insensitive word-boundary match for an exact seeded value.""" + return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) + + +def reports_exact_int(text: str, n: int) -> bool: + """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" + return bool(word_boundary(str(int(n))).search(text or "")) + + +def whole_answer_int(text: str) -> int | None: + """If the answer (or its last non-empty line) is exactly an integer, return it. + + Letters must not appear — only surrounding whitespace/punctuation is ignored — + so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading + minus** attached to the number is preserved (``-3`` → -3, not 3). + """ + + def _as_int(s: str) -> int | None: + # Collapse whitespace; then the whole string must be optional sign + digits + # with only non-word punctuation wrappers (prefix must not eat the sign). + compact = re.sub(r"\s+", "", s or "") + m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) + if m: + return int(m.group(1)) + return None + + blob = text or "" + v = _as_int(blob) + if v is not None: + return v + lines = [ln for ln in blob.splitlines() if ln.strip()] + if lines: + return _as_int(lines[-1]) + return None + + +def reports_contract_int(text: str, truth: int) -> bool: + """True when final text reports ``truth`` via the explicit ``count: N`` contract. + + 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding + whitespace allowed). Use the **last** match; require signed equality with + ``truth``. + 2. Fallback: whole-answer / last-line bare integer (:func:`whole_answer_int`). + 3. No match at all → False (ignoring an explicit format instruction is a fail). + """ + last: int | None = None + for line in (text or "").splitlines(): + m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) + if m: + last = int(m.group(1)) + if last is not None: + return last == int(truth) + whole = whole_answer_int(text) + if whole is not None: + return whole == int(truth) + return False + + +def contract_values(text: str, field: str) -> list[str]: + """Return non-empty values from exact ``field: value`` contract lines. + + The field name is case-insensitive, as with :func:`reports_contract_int`, + while the value is preserved for exact comparison. Prose, bullets, inline + mentions, and malformed/empty contract lines are ignored. + """ + values: list[str] = [] + pattern = re.compile(rf"\s*{re.escape(field)}:\s*(.*?)\s*", flags=re.IGNORECASE) + for line in (text or "").splitlines(): + match = pattern.fullmatch(line) + if match and match.group(1): + values.append(match.group(1)) + return values + + +def reports_contract_value(text: str, field: str, truth: str) -> bool: + """True when exactly one ``field: value`` line equals ``truth`` exactly.""" + return contract_values(text, field) == [str(truth)] + + +def reports_contract_values(text: str, field: str, truths: list[str] | tuple[str, ...]) -> bool: + """True when contract lines equal the expected value multiset. + + Ordering is deliberately ignored: the output contract defines one exact + fact per line, not a presentation order. Missing, duplicate, or extra field + lines fail. + """ + return Counter(contract_values(text, field)) == Counter(str(value) for value in truths) + + +def get_final_text(run: dict[str, Any]) -> str: + return run.get("final_text") or "" + + +__all__ = [ + "contract_values", + "get_final_text", + "reports_contract_int", + "reports_contract_value", + "reports_contract_values", + "reports_exact_int", + "whole_answer_int", + "word_boundary", +] diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py new file mode 100644 index 00000000..f1f2c10e --- /dev/null +++ b/evals/tasks/catalog.py @@ -0,0 +1,117 @@ +"""Task catalog assembly, lookup, authorship, and fingerprinting.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from evals.tasks.cross import CROSS_TASKS +from evals.tasks.debias import DEBIAS_TASKS +from evals.tasks.read import READ_TASKS +from evals.tasks.schema import SCHEMA_TASKS +from evals.tasks.write import WRITE_TASKS + +EXPECTED_TASK_IDS = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) + +# Preserve the historical catalog order exactly: R7 was added after C1/C2. +TASKS: list[dict[str, Any]] = [ + *READ_TASKS[:6], + *WRITE_TASKS, + *SCHEMA_TASKS, + *CROSS_TASKS, + READ_TASKS[6], + *DEBIAS_TASKS, +] +if tuple(task["id"] for task in TASKS) != EXPECTED_TASK_IDS: + raise RuntimeError("assembled task order changed; battery/result compatibility would break") + +TASKS_BY_ID: dict[str, dict[str, Any]] = {task["id"]: task for task in TASKS} + + +def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: + """Return tasks filtered by id list (None = all).""" + if ids is None: + return list(TASKS) + missing = [i for i in ids if i not in TASKS_BY_ID] + if missing: + raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") + return [TASKS_BY_ID[i] for i in ids] + + +def task_author(task: dict[str, Any]) -> str: + """Return the task author; default ``claude`` when the key is absent.""" + return str(task.get("author") or "claude") + + +def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: + """Stable short hash of the task battery used for a run. + + SHA-256 (first 12 hex chars) over a canonical serialization of every task + sorted by id: id, prompt, sorted optimal/alternate tools, and optimal_calls. + + Ceilings (intentionally *not* covered by the hash): + - Verifier functions and ``needs`` fixtures do not alter the fingerprint — + prompt/tool-set drift is the stability signal, not seed/verify logic. + - The hash covers the *selected* task list: ``--tasks`` subsets produce + different fingerprints than a full-catalog run. + """ + src = list(TASKS if tasks is None else tasks) + payload: list[dict[str, Any]] = [] + for t in sorted(src, key=lambda x: str(x.get("id") or "")): + payload.append( + { + "id": t.get("id"), + "prompt": t.get("prompt"), + "optimal_tools": sorted(t.get("optimal_tools") or []), + "alternate_tools": sorted(t.get("alternate_tools") or []), + "optimal_calls": t.get("optimal_calls"), + } + ) + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + +__all__ = [ + "EXPECTED_TASK_IDS", + "TASKS", + "TASKS_BY_ID", + "battery_fingerprint", + "get_tasks", + "task_author", +] diff --git a/evals/tasks/common.py b/evals/tasks/common.py deleted file mode 100644 index e52edf49..00000000 --- a/evals/tasks/common.py +++ /dev/null @@ -1,307 +0,0 @@ -"""Shared prompt, matching, and API lookup machinery for eval tasks.""" - -from __future__ import annotations - -import re -import string -from collections import Counter -from typing import Any - -from plane.errors.errors import HttpError -from plane.models.query_params import WorkItemQueryParams - - -class TaskSkipped(Exception): - """Verifier signals that this task-rep should be recorded as skipped, not failed.""" - - def __init__(self, reason: str) -> None: - super().__init__(reason) - self.reason = reason - - -class PromptBindError(RuntimeError): - """Live prompt could not bind required seed IDs (classified as infra_seed).""" - - -def format_task_prompt( - task: dict[str, Any], - ctx: dict[str, Any] | None = None, - *, - strict: bool = False, -) -> str: - """Render a task prompt with seed-bound placeholders. - - Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand - the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an - optional ``prompt_bind(ctx) -> dict`` callable on the task dict. - - When ``strict=True`` (live runs), empty-string values or binder exceptions - raise ``PromptBindError`` so the harness records ``infra_seed`` rather than - sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and - fills missing keys with explicit ```` markers. - """ - tpl = str(task.get("prompt") or "") - fields: dict[str, Any] = { - "project": (ctx or {}).get("project_name") or "EVAL deadbeef", - } - binder = task.get("prompt_bind") - if callable(binder) and ctx is not None: - try: - extra = binder(ctx) or {} - except Exception as exc: - if strict: - raise PromptBindError( - f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" - ) from exc - extra = {} - if isinstance(extra, dict): - for key, val in extra.items(): - if val is None: - if strict: - raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") - continue - text = str(val).strip() - if not text: - if strict: - raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") - continue - fields[key] = text - # Collect required placeholders from the template. - required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] - for name in required: - if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): - continue - if strict: - raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") - fields.setdefault(name, f"<{name}>") - return tpl.format(**fields) - - -def word_boundary(value: str) -> re.Pattern[str]: - """Compile a case-insensitive word-boundary match for an exact seeded value.""" - return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) - - -def reports_exact_int(text: str, n: int) -> bool: - """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" - return bool(word_boundary(str(int(n))).search(text or "")) - - -def whole_answer_int(text: str) -> int | None: - """If the answer (or its last non-empty line) is exactly an integer, return it. - - Letters must not appear — only surrounding whitespace/punctuation is ignored — - so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading - minus** attached to the number is preserved (``-3`` → -3, not 3). - """ - - def _as_int(s: str) -> int | None: - # Collapse whitespace; then the whole string must be optional sign + digits - # with only non-word punctuation wrappers (prefix must not eat the sign). - compact = re.sub(r"\s+", "", s or "") - m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) - if m: - return int(m.group(1)) - return None - - blob = text or "" - v = _as_int(blob) - if v is not None: - return v - lines = [ln for ln in blob.splitlines() if ln.strip()] - if lines: - return _as_int(lines[-1]) - return None - - -def reports_contract_int(text: str, truth: int) -> bool: - """True when final text reports ``truth`` via the explicit ``count: N`` contract. - - 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding - whitespace allowed). Use the **last** match; require signed equality with - ``truth``. - 2. Fallback: whole-answer / last-line bare integer (:func:`whole_answer_int`). - 3. No match at all → False (ignoring an explicit format instruction is a fail). - """ - last: int | None = None - for line in (text or "").splitlines(): - m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) - if m: - last = int(m.group(1)) - if last is not None: - return last == int(truth) - whole = whole_answer_int(text) - if whole is not None: - return whole == int(truth) - return False - - -def contract_values(text: str, field: str) -> list[str]: - """Return non-empty values from exact ``field: value`` contract lines. - - The field name is case-insensitive, as with :func:`reports_contract_int`, - while the value is preserved for exact comparison. Prose, bullets, inline - mentions, and malformed/empty contract lines are ignored. - """ - values: list[str] = [] - pattern = re.compile(rf"\s*{re.escape(field)}:\s*(.*?)\s*", flags=re.IGNORECASE) - for line in (text or "").splitlines(): - match = pattern.fullmatch(line) - if match and match.group(1): - values.append(match.group(1)) - return values - - -def reports_contract_value(text: str, field: str, truth: str) -> bool: - """True when exactly one ``field: value`` line equals ``truth`` exactly.""" - return contract_values(text, field) == [str(truth)] - - -def reports_contract_values(text: str, field: str, truths: list[str] | tuple[str, ...]) -> bool: - """True when contract lines equal the expected value multiset. - - Ordering is deliberately ignored: the output contract defines one exact - fact per line, not a presentation order. Missing, duplicate, or extra field - lines fail. - """ - return Counter(contract_values(text, field)) == Counter(str(value) for value in truths) - - -def as_id(obj: Any) -> str | None: - if obj is None: - return None - if isinstance(obj, str): - return obj - return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) - - -def ids(items: Any) -> set[str]: - out: set[str] = set() - for item in items or []: - i = as_id(item) - if i: - out.add(str(i)) - return out - - -def find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: - """Return all work items with exact name, newest first (by created_at).""" - matches: list[Any] = [] - cursor = None - while True: - params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) - page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) - for item in page.results or []: - if (item.name or "").strip() == name: - matches.append(item) - if not page.next_page_results: - break - cursor = page.next_cursor - - def _created_key(item: Any) -> str: - return str(getattr(item, "created_at", None) or "") - - matches.sort(key=_created_key, reverse=True) - return matches - - -def find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: - """Locate a work item by exact name; when duplicates exist, prefer the newest.""" - matches = find_items_by_name(plane, workspace_slug, project_id, name) - return matches[0] if matches else None - - -def state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: - """Resolve a state UUID or expanded object to its display name.""" - if state_ref is None: - return None - if hasattr(state_ref, "name") and state_ref.name: - return str(state_ref.name) - if isinstance(state_ref, dict) and state_ref.get("name"): - return str(state_ref["name"]) - state_id = as_id(state_ref) - if not state_id: - return None - try: - state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) - return state.name - except HttpError as exc: - if exc.status_code not in (404, 405): - raise - # Fall back to listing states and matching by id. - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - results = page.results if hasattr(page, "results") else page - for s in results or []: - if str(s.id) == str(state_id): - return s.name - return None - - -def state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: - if state_ref is None: - return None - if hasattr(state_ref, "group") and state_ref.group: - return str(state_ref.group) - if isinstance(state_ref, dict) and state_ref.get("group"): - return str(state_ref["group"]) - state_id = as_id(state_ref) - if not state_id: - return None - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - for s in page.results or []: - if str(s.id) == str(state_id): - return getattr(s, "group", None) - return None - - -def is_not_found(exc: BaseException) -> bool: - return isinstance(exc, HttpError) and exc.status_code in (404, 405) - - -def get_final_text(run: dict[str, Any]) -> str: - return run.get("final_text") or "" - - -def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: - """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" - page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} - n = 0 - cursor = None - while True: - params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) - resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) - for item in resp.results or []: - if (getattr(item, "priority", None) or "").lower() != "urgent": - continue - sid = as_id(item.state) - if sid and str(sid) in closed_ids: - continue - n += 1 - if not resp.next_page_results: - break - cursor = resp.next_cursor - return n - - -__all__ = [ - "TaskSkipped", - "PromptBindError", - "format_task_prompt", - "word_boundary", - "reports_exact_int", - "whole_answer_int", - "reports_contract_int", - "contract_values", - "reports_contract_value", - "reports_contract_values", - "as_id", - "ids", - "find_items_by_name", - "find_item_by_name", - "state_name", - "state_group", - "is_not_found", - "get_final_text", - "count_open_urgent", -] diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py index df56b1ef..1422b080 100644 --- a/evals/tasks/cross.py +++ b/evals/tasks/cross.py @@ -12,15 +12,13 @@ RELEASE_CHANGELOG_TEXT, RELEASE_NAME, ) -from evals.tasks.common import ( - as_id, +from evals.tasks.answers import ( contract_values, - find_item_by_name, get_final_text, - ids, reports_contract_value, reports_contract_values, ) +from evals.tasks.lookups import as_id, find_item_by_name, ids async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 41fae19f..9b9ed5c3 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -17,15 +17,14 @@ W3_TITLE, W8_TITLE, ) -from evals.tasks.common import ( +from evals.tasks.answers import ( contract_values, get_final_text, - ids, reports_contract_int, reports_contract_value, reports_contract_values, - state_name, ) +from evals.tasks.lookups import ids, state_name I1_TITLE = R1_TITLE diff --git a/evals/tasks/lookups.py b/evals/tasks/lookups.py new file mode 100644 index 00000000..8b658041 --- /dev/null +++ b/evals/tasks/lookups.py @@ -0,0 +1,133 @@ +"""Plane reads used by task verifiers to establish truth.""" + +from __future__ import annotations + +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.query_params import WorkItemQueryParams + + +def as_id(obj: Any) -> str | None: + if obj is None: + return None + if isinstance(obj, str): + return obj + return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) + + +def ids(items: Any) -> set[str]: + out: set[str] = set() + for item in items or []: + i = as_id(item) + if i: + out.add(str(i)) + return out + + +def find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: + """Return all work items with exact name, newest first (by created_at).""" + matches: list[Any] = [] + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in page.results or []: + if (item.name or "").strip() == name: + matches.append(item) + if not page.next_page_results: + break + cursor = page.next_cursor + + def _created_key(item: Any) -> str: + return str(getattr(item, "created_at", None) or "") + + matches.sort(key=_created_key, reverse=True) + return matches + + +def find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: + """Locate a work item by exact name; when duplicates exist, prefer the newest.""" + matches = find_items_by_name(plane, workspace_slug, project_id, name) + return matches[0] if matches else None + + +def state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + """Resolve a state UUID or expanded object to its display name.""" + if state_ref is None: + return None + if hasattr(state_ref, "name") and state_ref.name: + return str(state_ref.name) + if isinstance(state_ref, dict) and state_ref.get("name"): + return str(state_ref["name"]) + state_id = as_id(state_ref) + if not state_id: + return None + try: + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return state.name + except HttpError as exc: + if exc.status_code not in (404, 405): + raise + # Fall back to listing states and matching by id. + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + results = page.results if hasattr(page, "results") else page + for s in results or []: + if str(s.id) == str(state_id): + return s.name + return None + + +def state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + if state_ref is None: + return None + if hasattr(state_ref, "group") and state_ref.group: + return str(state_ref.group) + if isinstance(state_ref, dict) and state_ref.get("group"): + return str(state_ref["group"]) + state_id = as_id(state_ref) + if not state_id: + return None + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + for s in page.results or []: + if str(s.id) == str(state_id): + return getattr(s, "group", None) + return None + + +def is_not_found(exc: BaseException) -> bool: + return isinstance(exc, HttpError) and exc.status_code in (404, 405) + + +def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: + """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} + n = 0 + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in resp.results or []: + if (getattr(item, "priority", None) or "").lower() != "urgent": + continue + sid = as_id(item.state) + if sid and str(sid) in closed_ids: + continue + n += 1 + if not resp.next_page_results: + break + cursor = resp.next_cursor + return n + + +__all__ = [ + "as_id", + "count_open_urgent", + "find_item_by_name", + "find_items_by_name", + "ids", + "is_not_found", + "state_group", + "state_name", +] diff --git a/evals/tasks/prompts.py b/evals/tasks/prompts.py new file mode 100644 index 00000000..88be89eb --- /dev/null +++ b/evals/tasks/prompts.py @@ -0,0 +1,67 @@ +"""Task prompt binding.""" + +from __future__ import annotations + +import string +from typing import Any + + +class PromptBindError(RuntimeError): + """Live prompt could not bind required seed IDs (classified as infra_seed).""" + + +def format_task_prompt( + task: dict[str, Any], + ctx: dict[str, Any] | None = None, + *, + strict: bool = False, +) -> str: + """Render a task prompt with seed-bound placeholders. + + Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand + the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an + optional ``prompt_bind(ctx) -> dict`` callable on the task dict. + + When ``strict=True`` (live runs), empty-string values or binder exceptions + raise ``PromptBindError`` so the harness records ``infra_seed`` rather than + sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and + fills missing keys with explicit ```` markers. + """ + tpl = str(task.get("prompt") or "") + fields: dict[str, Any] = { + "project": (ctx or {}).get("project_name") or "EVAL deadbeef", + } + binder = task.get("prompt_bind") + if callable(binder) and ctx is not None: + try: + extra = binder(ctx) or {} + except Exception as exc: + if strict: + raise PromptBindError( + f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" + ) from exc + extra = {} + if isinstance(extra, dict): + for key, val in extra.items(): + if val is None: + if strict: + raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") + continue + text = str(val).strip() + if not text: + if strict: + raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") + continue + fields[key] = text + # Collect required placeholders from the template. + required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] + for name in required: + if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): + continue + if strict: + raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") + fields.setdefault(name, f"<{name}>") + return tpl.format(**fields) + + +__all__ = ["PromptBindError", "format_task_prompt"] diff --git a/evals/tasks/read.py b/evals/tasks/read.py index c0f7f0b8..cc44ece6 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -5,16 +5,14 @@ from typing import Any from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, R5_TITLE -from evals.tasks.common import ( +from evals.tasks.answers import ( contract_values, - count_open_urgent, - find_item_by_name, get_final_text, reports_contract_int, reports_contract_value, reports_contract_values, - state_name, ) +from evals.tasks.lookups import count_open_urgent, find_item_by_name, state_name async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py index 8984bff2..e9800ba2 100644 --- a/evals/tasks/schema.py +++ b/evals/tasks/schema.py @@ -8,7 +8,8 @@ from plane.models.enums import PropertyType from evals.seed import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE -from evals.tasks.common import TaskSkipped, as_id, find_item_by_name, is_not_found +from evals.tasks.lookups import as_id, find_item_by_name, is_not_found +from evals.tasks.skip import TaskSkipped async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: diff --git a/evals/tasks/skip.py b/evals/tasks/skip.py new file mode 100644 index 00000000..fc87fde2 --- /dev/null +++ b/evals/tasks/skip.py @@ -0,0 +1,12 @@ +"""Task skip signal.""" + + +class TaskSkipped(Exception): + """Verifier signals that this task-rep should be recorded as skipped, not failed.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +__all__ = ["TaskSkipped"] diff --git a/evals/tasks/write.py b/evals/tasks/write.py index b988d986..5c08b8a2 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -19,14 +19,14 @@ W7_URL, W8_TITLE, ) -from evals.tasks.common import ( +from evals.tasks.answers import word_boundary +from evals.tasks.lookups import ( find_item_by_name, find_items_by_name, ids, is_not_found, state_group, state_name, - word_boundary, ) diff --git a/tests/test_evals_catalog.py b/tests/test_evals_catalog.py index a58be195..6cbe1cf4 100644 --- a/tests/test_evals_catalog.py +++ b/tests/test_evals_catalog.py @@ -10,7 +10,7 @@ from evals import tasks as tasks_mod from evals.cli import cmd_dry_run, cmd_list, parse_args from evals.seed import seed_plan -from evals.tasks import TASKS, TASKS_BY_ID, get_tasks +from evals.tasks.catalog import TASKS, TASKS_BY_ID, get_tasks # DESIGN.md catalog ids (stable) + extras added for uncovered tool families. DESIGN_IDS = { @@ -126,7 +126,7 @@ def test_task_schema_invariants(): def test_debias_tasks_author(): - from evals.tasks import task_author + from evals.tasks.catalog import task_author for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: t = TASKS_BY_ID[tid] diff --git a/tests/test_evals_debias_verifiers.py b/tests/test_evals_debias_verifiers.py index 98238bb7..3182a1b9 100644 --- a/tests/test_evals_debias_verifiers.py +++ b/tests/test_evals_debias_verifiers.py @@ -14,7 +14,8 @@ import pytest from evals.seed import R1_TITLE, W2_TITLE, W8_TITLE -from evals.tasks import ( +from evals.tasks.cross import verify_c2 +from evals.tasks.debias import ( I1_TITLE, I3_TITLE, I4_TITLE, @@ -24,7 +25,6 @@ L4_PROP_DISPLAY, L4_PROP_VALUE, L5_TITLE, - verify_c2, verify_i1, verify_i2, verify_i3, @@ -35,11 +35,9 @@ verify_l3, verify_l4, verify_l5, - verify_r1, - verify_w2, - verify_w4, - verify_w8, ) +from evals.tasks.read import verify_r1 +from evals.tasks.write import verify_w2, verify_w4, verify_w8 class _Page: @@ -632,7 +630,7 @@ async def _go(): def test_reports_contract_int_unit(): """Direct unit cases for the contract helper.""" - from evals.tasks import reports_contract_int + from evals.tasks.answers import reports_contract_int assert reports_contract_int("count: 3", 3) is True assert reports_contract_int("count: 2", 3) is False @@ -647,7 +645,7 @@ def test_reports_contract_int_unit(): def test_exact_line_contract_helpers_unit(): - from evals.tasks import contract_values, reports_contract_value, reports_contract_values + from evals.tasks.answers import contract_values, reports_contract_value, reports_contract_values text = "prose mentions state Done\nSTATE: In Progress\nitem: B\nitem: A" assert contract_values(text, "state") == ["In Progress"] @@ -739,7 +737,7 @@ def __init__(self, count: int): def test_existing_r2_wrong_count_in_text_fails(): async def _go(): # verify_r2 counts open urgent via SDK; text must match that count. - from evals.tasks import verify_r2 as _vr2 + from evals.tasks.read import verify_r2 as _vr2 class Plane: def __init__(self): @@ -889,7 +887,8 @@ async def _go(): def test_prompt_bind_strict_empty_raises(): - from evals.tasks import TASKS_BY_ID, PromptBindError, format_task_prompt + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import PromptBindError, format_task_prompt t = TASKS_BY_ID["I1"] with pytest.raises(PromptBindError): @@ -897,7 +896,7 @@ def test_prompt_bind_strict_empty_raises(): def test_prompt_bind_strict_exception_raises(): - from evals.tasks import PromptBindError, format_task_prompt + from evals.tasks.prompts import PromptBindError, format_task_prompt def boom(_ctx): raise RuntimeError("seed broken") @@ -912,7 +911,8 @@ def boom(_ctx): def test_prompt_bind_dry_run_markers(): - from evals.tasks import TASKS_BY_ID, format_task_prompt + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt t = TASKS_BY_ID["I1"] text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) @@ -921,7 +921,8 @@ def test_prompt_bind_dry_run_markers(): def test_prompt_bind_strict_success(): - from evals.tasks import TASKS_BY_ID, format_task_prompt + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt t = TASKS_BY_ID["I1"] text = format_task_prompt( @@ -1057,7 +1058,7 @@ def test_l2_activity_gate_raises_when_empty(): from types import SimpleNamespace from evals.seed import R5_TITLE, _gate_activity_worker - from evals.tasks import TaskSkipped + from evals.tasks.skip import TaskSkipped class Plane: work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index a2386d8a..6c404f9a 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -28,7 +28,8 @@ ) from evals.runner import live as runner_live from evals.seed import create_project_with_identifier_retry, is_identifier_collision -from evals.tasks import TaskSkipped, battery_fingerprint, task_author +from evals.tasks.catalog import battery_fingerprint, task_author +from evals.tasks.skip import TaskSkipped # Pinned hash of the fixed synthetic catalog in test_battery_fingerprint_stable_and_sensitive. # Recompute only if the serialization format of battery_fingerprint changes deliberately. @@ -696,7 +697,7 @@ def test_battery_fingerprint_stable_and_sensitive(): def test_battery_fingerprint_catalog_is_nonempty(): - from evals.tasks import TASKS + from evals.tasks.catalog import TASKS fp = battery_fingerprint() assert len(fp) == 12 @@ -705,7 +706,7 @@ def test_battery_fingerprint_catalog_is_nonempty(): def test_battery_fingerprint_changes_with_new_debias_tasks(): """Adding I/L content must change the catalog fingerprint (content hash).""" - from evals.tasks import TASKS, TASKS_BY_ID + from evals.tasks.catalog import TASKS, TASKS_BY_ID full = battery_fingerprint() without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] diff --git a/tests/test_evals_output_contracts.py b/tests/test_evals_output_contracts.py index 3e36d596..483fe106 100644 --- a/tests/test_evals_output_contracts.py +++ b/tests/test_evals_output_contracts.py @@ -7,7 +7,8 @@ from typing import Any from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES -from evals.tasks import verify_c2, verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.cross import verify_c2 +from evals.tasks.read import verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 class _Page: diff --git a/tests/test_evals_verifiers.py b/tests/test_evals_verifiers.py index 0e02fbb5..8b53038e 100644 --- a/tests/test_evals_verifiers.py +++ b/tests/test_evals_verifiers.py @@ -26,18 +26,10 @@ W7_URL, W8_TITLE, ) -from evals.tasks import ( - verify_c1, - verify_r3, - verify_s3, - verify_s5, - verify_w3, - verify_w4, - verify_w5, - verify_w6, - verify_w7, - verify_w8, -) +from evals.tasks.cross import verify_c1 +from evals.tasks.read import verify_r3 +from evals.tasks.schema import verify_s3, verify_s5 +from evals.tasks.write import verify_w3, verify_w4, verify_w5, verify_w6, verify_w7, verify_w8 # --------------------------------------------------------------------------- # Tiny helpers From 6feffb71e2170ab9074931da2247d1872aa6ac88 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 18:24:49 +0530 Subject: [PATCH 21/93] Give the report summary a type instead of a magic key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit summarize() returned a dict of task id to dict, with the run-level totals smuggled in under the key "_meta". Seven places across table.py, command.py and compare.py had to remember that key existed, and forgetting one would either treat the totals as a task or drop them. Every field read was an untyped .get with a default, so a misspelled field name read as a missing value instead of failing. TaskSummary and Summary replace it. unstable, success, unstable_task_ids, and unstable_tasks are derived properties rather than stored fields, so the count of unstable tasks can no longer disagree with the list of them. No metric changed. Proven by byte-comparing all three report modes — single, table, and markdown table — over a real 34-task battery across three columns before and after: identical output. --- evals/report/__init__.py | 4 +- evals/report/command.py | 8 +- evals/report/compare.py | 24 +++--- evals/report/summary.py | 135 ++++++++++++++++++++++----------- evals/report/table.py | 77 +++++++++---------- tests/test_evals_hardening.py | 43 ++++------- tests/test_evals_report_ops.py | 65 ++++++++-------- 7 files changed, 191 insertions(+), 165 deletions(-) diff --git a/evals/report/__init__.py b/evals/report/__init__.py index 34785668..8a1e7a3e 100644 --- a/evals/report/__init__.py +++ b/evals/report/__init__.py @@ -12,7 +12,7 @@ read_result, ) from .statistics import iqr, median, percentile, sign_test_pvalue, wilson_interval -from .summary import ResultTokensMode, noise_floor_statement, result_tokens_mode, summarize +from .summary import ResultTokensMode, Summary, TaskSummary, noise_floor_statement, result_tokens_mode, summarize from .table import ( build_multi_surface_table, format_multi_rep_surface_cell, @@ -32,6 +32,8 @@ "DedupeMode", "ResultRow", "ResultTokensMode", + "Summary", + "TaskSummary", "ab_compare", "build_multi_surface_table", "dedupe_rows_latest", diff --git a/evals/report/command.py b/evals/report/command.py index e1db86ae..fb194d1b 100644 --- a/evals/report/command.py +++ b/evals/report/command.py @@ -81,11 +81,9 @@ def main(argv: list[str] | None = None) -> int: path = paths[0] rows = load_rows(path, dedupe=dedupe) summary = summarize(rows) - task_keys = [key for key in summary if key != "_meta"] - if not task_keys: - infrastructure_errors = (summary.get("_meta") or {}).get("infra_errors", 0) - if infrastructure_errors: - print(f"infra errors: {infrastructure_errors}") + if not summary.tasks: + if summary.infra_errors: + print(f"infra errors: {summary.infra_errors}") print(f"(no non-skipped / non-error rows in {path})") return 0 print_table(summary, f"Summary: {path}") diff --git a/evals/report/compare.py b/evals/report/compare.py index 3c360424..b1a06ca4 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -55,8 +55,6 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: } ) - meta_a = summary_a.get("_meta") or {} - meta_b = summary_b.get("_meta") or {} return { "summary_a": summary_a, "summary_b": summary_b, @@ -64,23 +62,23 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: "median_delta": median(deltas), "sign_test_p": sign_test_pvalue(deltas), "n_paired": len(deltas), - "multi_rep": bool(meta_a.get("multi_rep") or meta_b.get("multi_rep")), - "unstable_a": int(meta_a.get("unstable_tasks") or 0), - "unstable_b": int(meta_b.get("unstable_tasks") or 0), + "multi_rep": summary_a.multi_rep or summary_b.multi_rep, + "unstable_a": summary_a.unstable_tasks, + "unstable_b": summary_b.unstable_tasks, "success_a": { - "k": int(meta_a.get("aggregate_k") or 0), - "n": int(meta_a.get("aggregate_n") or 0), + "k": summary_a.aggregate_k, + "n": summary_a.aggregate_n, "wilson": ( - float(meta_a.get("aggregate_wilson_lo") or 0.0), - float(meta_a.get("aggregate_wilson_hi") or 0.0), + summary_a.aggregate_wilson_lo, + summary_a.aggregate_wilson_hi, ), }, "success_b": { - "k": int(meta_b.get("aggregate_k") or 0), - "n": int(meta_b.get("aggregate_n") or 0), + "k": summary_b.aggregate_k, + "n": summary_b.aggregate_n, "wilson": ( - float(meta_b.get("aggregate_wilson_lo") or 0.0), - float(meta_b.get("aggregate_wilson_hi") or 0.0), + summary_b.aggregate_wilson_lo, + summary_b.aggregate_wilson_hi, ), }, } diff --git a/evals/report/summary.py b/evals/report/summary.py index 2ce90657..fd8405de 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -3,7 +3,8 @@ from __future__ import annotations from collections import defaultdict -from typing import Any, Literal +from dataclasses import dataclass +from typing import Literal from evals.results import TaskResult from evals.tasks import TASKS_BY_ID @@ -14,6 +15,59 @@ ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] +@dataclass(slots=True) +class TaskSummary: + task_id: str + n: int + k: int + wilson_lo: float + wilson_hi: float + med_calls: float | None + calls_min: float | None + calls_max: float | None + calls_q1: float | None + calls_q3: float | None + optimal_calls: int | None + mispick_rate: float + errored_calls: int + capped: int + harness_err: int + infra_err: int + med_result_tokens: float | None + p95_result_tokens: float | None + result_tokens_mode: ResultTokensMode + med_cum_input: float | None + + @property + def unstable(self) -> bool: + """True when repetitions of this task disagreed on pass/fail.""" + return self.n > 1 and 0 < self.k < self.n + + @property + def success(self) -> str: + return f"{self.k}/{self.n}" if self.n else "0/0" + + +@dataclass(slots=True) +class Summary: + tasks: dict[str, TaskSummary] + infra_errors: int + aggregate_k: int + aggregate_n: int + aggregate_wilson_lo: float + aggregate_wilson_hi: float + multi_rep: bool + result_tokens_mode: ResultTokensMode + + @property + def unstable_task_ids(self) -> list[str]: + return [task_id for task_id, task in self.tasks.items() if task.unstable] + + @property + def unstable_tasks(self) -> int: + return len(self.unstable_task_ids) + + def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: """Classify token counts without treating unmarked legacy data as measured.""" labels: set[str] = set() @@ -44,13 +98,13 @@ def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: return "mixed" -def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: +def summarize(rows: list[ResultRow]) -> Summary: """Aggregate per-task metrics. Rows with ``error_class`` starting ``infra_`` are excluded from success-rate - denominators and counted separately as ``infra_errors`` (total on the returned - dict under the special key ``_meta``). Other non-null ``error`` rows remain - harness errors (excluded from success, counted in ``harness_err``). + denominators and counted separately as ``infra_errors``. Other non-null + ``error`` rows remain harness errors (excluded from success, counted in + ``harness_err``). """ by_task: dict[str, list[TaskResult]] = defaultdict(list) harness_errors_by_task: dict[str, int] = defaultdict(int) @@ -77,14 +131,13 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: # Include tasks that only had harness/infra errors so columns stay visible. all_task_ids = sorted(set(by_task) | set(harness_errors_by_task) | set(infrastructure_errors_by_task)) - output: dict[str, dict[str, Any]] = {} + output: dict[str, TaskSummary] = {} total_passes = 0 total_repetitions = 0 for task_id in all_task_ids: task_results = by_task.get(task_id, []) repetition_count = len(task_results) pass_count = sum(1 for row in task_results if row.success) - unstable = repetition_count > 1 and 0 < pass_count < repetition_count total_passes += pass_count total_repetitions += repetition_count lower, upper = wilson_interval(pass_count, repetition_count) if repetition_count else (0.0, 0.0) @@ -108,45 +161,41 @@ def summarize(rows: list[ResultRow]) -> dict[str, dict[str, Any]]: result_tokens.append(float(call.result_tokens)) capped = sum(1 for row in task_results if row.hit_max_iterations or row.stop_reason == "max_tokens") cumulative_inputs = [float(row.cum_input_tokens or 0) for row in task_results] - output[task_id] = { - "n": repetition_count, - "k": pass_count, - "success": f"{pass_count}/{repetition_count}" if repetition_count else "0/0", - "unstable": unstable, - "wilson_lo": lower, - "wilson_hi": upper, - "med_calls": median_calls, - "calls_min": minimum_calls, - "calls_max": maximum_calls, - "calls_q1": first_quartile, - "calls_q3": third_quartile, - "optimal_calls": optimal, - "mispick_rate": (mispicks / total_calls) if total_calls else 0.0, - "errored_calls": errored_calls, - "capped": capped, - "harness_err": harness_errors_by_task.get(task_id, 0), - "infra_err": infrastructure_errors_by_task.get(task_id, 0), - "med_result_tokens": median(result_tokens), - "p95_result_tokens": percentile(result_tokens, 0.95), - "result_tokens_mode": result_tokens_mode(task_results), - "med_cum_input": median(cumulative_inputs), - } + output[task_id] = TaskSummary( + task_id=task_id, + n=repetition_count, + k=pass_count, + wilson_lo=lower, + wilson_hi=upper, + med_calls=median_calls, + calls_min=minimum_calls, + calls_max=maximum_calls, + calls_q1=first_quartile, + calls_q3=third_quartile, + optimal_calls=optimal, + mispick_rate=(mispicks / total_calls) if total_calls else 0.0, + errored_calls=errored_calls, + capped=capped, + harness_err=harness_errors_by_task.get(task_id, 0), + infra_err=infrastructure_errors_by_task.get(task_id, 0), + med_result_tokens=median(result_tokens), + p95_result_tokens=percentile(result_tokens, 0.95), + result_tokens_mode=result_tokens_mode(task_results), + med_cum_input=median(cumulative_inputs), + ) aggregate_lower, aggregate_upper = ( wilson_interval(total_passes, total_repetitions) if total_repetitions else (0.0, 0.0) ) - unstable_task_ids = sorted(task_id for task_id, values in output.items() if values.get("unstable")) - output["_meta"] = { - "infra_errors": infrastructure_errors, - "aggregate_k": total_passes, - "aggregate_n": total_repetitions, - "aggregate_wilson_lo": aggregate_lower, - "aggregate_wilson_hi": aggregate_upper, - "multi_rep": any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), - "unstable_task_ids": unstable_task_ids, - "unstable_tasks": len(unstable_task_ids), - "result_tokens_mode": result_tokens_mode([row for task_results in by_task.values() for row in task_results]), - } - return output + return Summary( + tasks=output, + infra_errors=infrastructure_errors, + aggregate_k=total_passes, + aggregate_n=total_repetitions, + aggregate_wilson_lo=aggregate_lower, + aggregate_wilson_hi=aggregate_upper, + multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), + result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), + ) def noise_floor_statement(unstable_tasks: int) -> str: diff --git a/evals/report/table.py b/evals/report/table.py index 2f0237e4..d327d9af 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -12,7 +12,7 @@ from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import wilson_interval -from .summary import noise_floor_statement +from .summary import Summary, noise_floor_statement def format_number(value: float | None, digits: int = 1) -> str: @@ -32,35 +32,32 @@ def format_result_tokens(value: float | None, mode: str) -> str: return f"{result_tokens_marker(mode)}{formatted}" -def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: - meta = summary.get("_meta") or {} +def print_table(summary: Summary, title: str) -> None: print(title) - token_mode = str(meta.get("result_tokens_mode") or "unavailable") + token_mode = summary.result_tokens_mode if token_mode == "estimated": print("result-token columns marked ~: entirely estimated from result characters") elif token_mode == "mixed": print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") elif token_mode == "unlabeled": print("result-token columns marked ?: include legacy values with unknown measurement status") - if meta.get("infra_errors"): - print(f"infra errors: {meta['infra_errors']}") - aggregate_count = int(meta.get("aggregate_n") or 0) + if summary.infra_errors: + print(f"infra errors: {summary.infra_errors}") + aggregate_count = summary.aggregate_n if aggregate_count: - aggregate_passes = int(meta.get("aggregate_k") or 0) - lower = float(meta.get("aggregate_wilson_lo") or 0.0) - upper = float(meta.get("aggregate_wilson_hi") or 0.0) + aggregate_passes = summary.aggregate_k + lower = summary.aggregate_wilson_lo + upper = summary.aggregate_wilson_hi rate = aggregate_passes / aggregate_count if aggregate_count else 0.0 print( f"aggregate success: {aggregate_passes}/{aggregate_count} ({rate:.1%}) Wilson95 [{lower:.2f},{upper:.2f}]" ) - multiple_repetitions = bool(meta.get("multi_rep")) + multiple_repetitions = summary.multi_rep if multiple_repetitions: - print(noise_floor_statement(int(meta.get("unstable_tasks") or 0))) + print(noise_floor_statement(summary.unstable_tasks)) # Multi-rep files keep the repetition-aware layout even when errors leave # only one completed result in every task's success-rate denominator. - show_variation = multiple_repetitions or any( - values.get("n", 0) > 1 for task_id, values in summary.items() if task_id != "_meta" - ) + show_variation = multiple_repetitions or any(task.n > 1 for task in summary.tasks.values()) token_marker = result_tokens_marker(token_mode) median_result_tokens_header = f"med_rtok{token_marker}" percentile_result_tokens_header = f"p95_rtok{token_marker}" @@ -85,39 +82,37 @@ def print_table(summary: dict[str, dict[str, Any]], title: str) -> None: ) print(header) print("-" * len(header)) - for task_id, values in summary.items(): - if task_id == "_meta": - continue - wilson = f"[{values['wilson_lo']:.2f},{values['wilson_hi']:.2f}]" - quartiles = f"{format_number(values['calls_q1'])}-{format_number(values['calls_q3'])}" - optimal = values["optimal_calls"] if values["optimal_calls"] is not None else "-" - task_token_mode = str(values.get("result_tokens_mode") or "unavailable") + for task_id, values in summary.tasks.items(): + wilson = f"[{values.wilson_lo:.2f},{values.wilson_hi:.2f}]" + quartiles = f"{format_number(values.calls_q1)}-{format_number(values.calls_q3)}" + optimal = values.optimal_calls if values.optimal_calls is not None else "-" + task_token_mode = values.result_tokens_mode if show_variation: - unstable = ("YES" if values.get("unstable") else "no") if multiple_repetitions else "" + unstable = ("YES" if values.unstable else "no") if multiple_repetitions else "" unstable_cell = f"{unstable:>8} " if multiple_repetitions else "" print( - f"{task_id:<6} {values['n']:>3} {values['success']:>8} {wilson:>16} " + f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " f"{unstable_cell}" - f"{format_number(values.get('calls_min')):>9} " - f"{format_number(values['med_calls']):>9} " - f"{format_number(values.get('calls_max')):>9} " - f"{optimal!s:>4} {quartiles:>11} {values['mispick_rate']:>7.1%} " - f"{values['errored_calls']:>4} {values['capped']:>6} " - f"{values['harness_err']:>5} {values.get('infra_err', 0):>5} " - f"{format_result_tokens(values['med_result_tokens'], task_token_mode):>9} " - f"{format_result_tokens(values['p95_result_tokens'], task_token_mode):>9} " - f"{format_number(values['med_cum_input'], 0):>10}" + f"{format_number(values.calls_min):>9} " + f"{format_number(values.med_calls):>9} " + f"{format_number(values.calls_max):>9} " + f"{optimal!s:>4} {quartiles:>11} {values.mispick_rate:>7.1%} " + f"{values.errored_calls:>4} {values.capped:>6} " + f"{values.harness_err:>5} {values.infra_err:>5} " + f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " + f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " + f"{format_number(values.med_cum_input, 0):>10}" ) else: print( - f"{task_id:<6} {values['n']:>3} {values['success']:>8} {wilson:>16} " - f"{format_number(values['med_calls']):>9} {optimal!s:>4} " - f"{quartiles:>11} {values['mispick_rate']:>7.1%} " - f"{values['errored_calls']:>4} {values['capped']:>6} " - f"{values['harness_err']:>5} {values.get('infra_err', 0):>5} " - f"{format_result_tokens(values['med_result_tokens'], task_token_mode):>9} " - f"{format_result_tokens(values['p95_result_tokens'], task_token_mode):>9} " - f"{format_number(values['med_cum_input'], 0):>10}" + f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " + f"{format_number(values.med_calls):>9} {optimal!s:>4} " + f"{quartiles:>11} {values.mispick_rate:>7.1%} " + f"{values.errored_calls:>4} {values.capped:>6} " + f"{values.harness_err:>5} {values.infra_err:>5} " + f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " + f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " + f"{format_number(values.med_cum_input, 0):>10}" ) diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 6c404f9a..6f5c13d5 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -744,38 +744,23 @@ def test_summarize_excludes_infra_errors_from_success(): {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, ] summary = summarize(rows) - assert summary["_meta"]["infra_errors"] == 2 - assert summary["R1"]["n"] == 2 # only non-infra, non-error rows - assert summary["R1"]["k"] == 1 - assert summary["R1"]["success"] == "1/2" - assert summary["R1"]["infra_err"] == 2 + assert summary.infra_errors == 2 + assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows + assert summary.tasks["R1"].k == 1 + assert summary.tasks["R1"].success == "1/2" + assert summary.tasks["R1"].infra_err == 2 assert is_infra_error_row(rows[1]) is True assert is_infra_error_row(rows[0]) is False def test_print_table_shows_infra_errors(capsys): - summary = { - "R1": { - "n": 1, - "k": 1, - "success": "1/1", - "wilson_lo": 0.2, - "wilson_hi": 1.0, - "med_calls": 1.0, - "calls_q1": 1.0, - "calls_q3": 1.0, - "optimal_calls": 1, - "mispick_rate": 0.0, - "errored_calls": 0, - "capped": 0, - "harness_err": 0, - "infra_err": 2, - "med_result_tokens": None, - "p95_result_tokens": None, - "med_cum_input": 0.0, - }, - "_meta": {"infra_errors": 2}, - } + summary = summarize( + [ + {"task_id": "R1", "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "error": "seed failed", "error_class": "infra_seed"}, + {"task_id": "R1", "error": "CLI failed", "error_class": "infra_cli"}, + ] + ) report_mod.print_table(summary, "Summary: test") out = capsys.readouterr().out assert "infra errors: 2" in out @@ -1125,5 +1110,5 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: # `skipped` is the discriminator, not `success` — a skip must leave the # success denominator empty rather than counting as a failed task. summary = summarize(load_rows(out)) - assert "L2" not in summary - assert summary["_meta"]["aggregate_n"] == 0 + assert "L2" not in summary.tasks + assert summary.aggregate_n == 0 diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py index fe889ec1..62dfffe1 100644 --- a/tests/test_evals_report_ops.py +++ b/tests/test_evals_report_ops.py @@ -161,12 +161,12 @@ def test_real_historical_rows_parse_and_report_with_backward_defaults(): assert [call.result_tokens for call in battery5.calls] == [315, 64] summary = summarize(rows) - assert summary["L3"]["success"] == "1/1" - assert summary["L3"]["med_calls"] == 1 - assert summary["L3"]["result_tokens_mode"] == "unavailable" - assert summary["R2"]["success"] == "1/1" - assert summary["R2"]["med_calls"] == 2 - assert summary["R2"]["result_tokens_mode"] == "estimated" + assert summary.tasks["L3"].success == "1/1" + assert summary.tasks["L3"].med_calls == 1 + assert summary.tasks["L3"].result_tokens_mode == "unavailable" + assert summary.tasks["R2"].success == "1/1" + assert summary.tasks["R2"].med_calls == 2 + assert summary.tasks["R2"].result_tokens_mode == "estimated" def test_dedupe_rows_latest_pure(): @@ -190,20 +190,19 @@ def test_summarize_aggregate_wilson_and_call_variance(): {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, ] s = summarize(rows) - assert s["R1"]["n"] == 3 - assert s["R1"]["k"] == 2 - assert s["R1"]["calls_min"] == 2.0 - assert s["R1"]["calls_max"] == 6.0 - assert s["R1"]["med_calls"] == 4.0 - assert s["R1"]["unstable"] is True - assert s["R2"]["unstable"] is False - meta = s["_meta"] - assert meta["aggregate_k"] == 3 - assert meta["aggregate_n"] == 4 - assert meta["multi_rep"] is True - assert meta["unstable_task_ids"] == ["R1"] - assert meta["unstable_tasks"] == 1 - assert 0.0 <= meta["aggregate_wilson_lo"] <= meta["aggregate_wilson_hi"] <= 1.0 + assert s.tasks["R1"].n == 3 + assert s.tasks["R1"].k == 2 + assert s.tasks["R1"].calls_min == 2.0 + assert s.tasks["R1"].calls_max == 6.0 + assert s.tasks["R1"].med_calls == 4.0 + assert s.tasks["R1"].unstable is True + assert s.tasks["R2"].unstable is False + assert s.aggregate_k == 3 + assert s.aggregate_n == 4 + assert s.multi_rep is True + assert s.unstable_task_ids == ["R1"] + assert s.unstable_tasks == 1 + assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): @@ -231,15 +230,15 @@ def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_pa summary = summarize(loaded) assert len(loaded) == 9 # distinct rep keys are not deduped away - assert summary["R1"]["success"] == "3/3" - assert summary["R1"]["unstable"] is False - assert summary["R2"]["success"] == "2/3" - assert summary["R2"]["wilson_lo"] == pytest.approx(0.2077, abs=1e-4) - assert summary["R2"]["wilson_hi"] == pytest.approx(0.9385, abs=1e-4) - assert summary["R2"]["unstable"] is True - assert summary["R3"]["success"] == "0/3" - assert summary["R3"]["unstable"] is False - assert summary["_meta"]["unstable_task_ids"] == ["R2"] + assert summary.tasks["R1"].success == "3/3" + assert summary.tasks["R1"].unstable is False + assert summary.tasks["R2"].success == "2/3" + assert summary.tasks["R2"].wilson_lo == pytest.approx(0.2077, abs=1e-4) + assert summary.tasks["R2"].wilson_hi == pytest.approx(0.9385, abs=1e-4) + assert summary.tasks["R2"].unstable is True + assert summary.tasks["R3"].success == "0/3" + assert summary.tasks["R3"].unstable is False + assert summary.unstable_task_ids == ["R2"] report_mod.print_table(summary, "Summary: multi.jsonl") output = capsys.readouterr().out @@ -280,8 +279,8 @@ def test_report_marks_entirely_estimated_result_token_columns(capsys): } ] summary = summarize(rows) - assert summary["_meta"]["result_tokens_mode"] == "estimated" - assert summary["R1"]["result_tokens_mode"] == "estimated" + assert summary.result_tokens_mode == "estimated" + assert summary.tasks["R1"].result_tokens_mode == "estimated" report_mod.print_table(summary, "estimated") output = capsys.readouterr().out @@ -310,8 +309,8 @@ def test_report_marks_mixed_measured_and_estimated_columns(capsys): }, ] summary = summarize(rows) - assert summary["_meta"]["result_tokens_mode"] == "mixed" - assert summary["R1"]["result_tokens_mode"] == "mixed" + assert summary.result_tokens_mode == "mixed" + assert summary.tasks["R1"].result_tokens_mode == "mixed" report_mod.print_table(summary, "mixed") output = capsys.readouterr().out From f35b498aaae321c1f7a132b799b38deb6bdbc90f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 18:40:21 +0530 Subject: [PATCH 22/93] Break the task repetition into named stages, behind a pinned taxonomy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _run_task_repetition was 183 lines nested six levels deep, and the error taxonomy ran through its else-chains. That taxonomy decides whether a failure is blamed on the agent or on the infrastructure, so it is the most consequential logic here and the least safe thing to restructure blind. So it was pinned first. Twelve characterization tests now drive run_live end to end and assert the recorded row for every branch: seed skip, seed failure, missing bug_type, prompt-bind failure, API-driver failure, CLI-driver failure, CLI timeout, error_max_turns staying a task failure, verifier skip, verifier error, external mispick nulling, and the success path's model identity. Five were new. infra_api had no test anywhere before this — the branch that decides whether a provider outage is recorded as a failed task. The function is now 73 lines of stage calls over seven helpers, each owning one stage and reporting whether to continue. Teardown still runs in a finally that covers every path, including skips and errors. Verified as behaviour-preserving three ways: the twelve tests pass unchanged against both the old and the new function; mutating three branches (api→cli, seed→task, max_turns→infra) each fails its own test, so they discriminate rather than merely pass; and a live run covering a success, a mutation, and a genuine env skip leaves the workspace empty. --- evals/runner/live.py | 386 ++++++++++++++++++++++------------ tests/test_evals_hardening.py | 313 ++++++++++++++++++++++++++- 2 files changed, 558 insertions(+), 141 deletions(-) diff --git a/evals/runner/live.py b/evals/runner/live.py index 243914da..fe6771d6 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -157,6 +157,224 @@ def _make_task_row( ) +def _seed_fixtures( + plane: Any, + task: dict[str, Any], + context: dict[str, Any], + *, + row: TaskResult, + repetition: int, +) -> bool: + """Seed one task and record fixture skips or infrastructure failures.""" + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + return False + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + ) + return False + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + row.skipped = reason + row.verify_note = reason + print(f" {task['id']} rep={repetition} SKIPPED: {reason}") + return False + return True + + +async def _drive_agent( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + context: dict[str, Any], + workspace_slug: str, + server_env: dict[str, str] | None, + row: TaskResult, + repetition: int, + is_api_driver: bool, +) -> TaskResult | None: + """Run the agent and classify launch or prompt failures.""" + # Agent wrap: API failures and CLI failures are infrastructure. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + return await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=context, + workspace_slug=workspace_slug, + optimal_tools=set(task["optimal_tools"]), + alternate_tools=set(task["alternate_tools"]), + server_env=server_env, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + ) + return None + except Exception as exc: + if is_api_driver: + agent_error_class = "infra_api" + else: + agent_error_class = "infra_cli" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = agent_error_class + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", + file=sys.stderr, + ) + return None + + +def _apply_agent_run( + row: TaskResult, + agent: TaskResult, + *, + model_alias: str, + requested_tier: str | None, + model_id: str | None, + external: bool, +) -> None: + """Copy agent metrics and restore the run-level model and server identity.""" + row.apply_agent_result(agent) + # Driver-level requested_model is the resolved ID. + # Restore run-level intent and retain both identities. + row.requested_model = model_alias + row.requested_tier = requested_tier + row.resolved_model = model_id + if external: + # Foreign tool names are not comparable to our catalog. + row.alternate_calls = None + row.out_of_set_calls = None + + +def _record_cli_infra_stop( + row: TaskResult, + agent: TaskResult, + *, + task: dict[str, Any], + repetition: int, + driver_name: str, +) -> bool: + """Record contained CLI infrastructure stops and block verification.""" + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.stop_reason + if not driver_name.endswith("-cli") or not is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): + return False + row.success = False + row.error_class = "infra_cli" + if stop_reason == "timeout": + row.error = _timeout_error_message(agent) + else: + notes = [note for note in agent.driver_notes if isinstance(note, str)] + detail = "; ".join(notes) if notes else str(stop_reason) + row.error = detail + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", + file=sys.stderr, + ) + return True + + +async def _verify_task( + plane: Any, + task: dict[str, Any], + context: dict[str, Any], + agent: TaskResult, + *, + row: TaskResult, + repetition: int, +) -> None: + """Run one verifier and record task outcomes or verifier failures.""" + verify = task["verify"] + try: + agent_row = agent.to_row() + ok, note = await verify( + plane, + context, + { + "final_text": agent.final_text, + "calls": agent_row["calls"], + }, + ) + row.success = bool(ok) + row.verify_note = note + print(f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}") + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[task]: {exc}", + file=sys.stderr, + ) + + +def _record_unexpected( + row: TaskResult, + exc: Exception, + *, + task: dict[str, Any], + repetition: int, + context: dict[str, Any], +) -> None: + """Record failures outside the seed, driver, and verifier boundaries.""" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print(f" {task['id']} rep={repetition} ERROR[task]: {exc}", file=sys.stderr) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + ) + + +def _remove_fixtures(plane: Any, context: dict[str, Any]) -> None: + """Remove task fixtures and retain the historical teardown diagnostics.""" + try: + teardown(plane, context) + except Exception as exc: + print(f" teardown error: {exc}", file=sys.stderr) + if context.get("project_name"): + print(f" orphaned project: {context['project_name']}", file=sys.stderr) + + async def _run_task_repetition( *, plane: Any, @@ -194,150 +412,40 @@ async def _run_task_repetition( server="external" if external else "local", ) try: - task_needs = set(task.get("needs") or set()) - # Seed wrap: TaskSkipped → skip; other failures → infra_seed. - try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", - file=sys.stderr, + if _seed_fixtures(plane, task, context, row=row, repetition=repetition): + agent = await _drive_agent( + driver=driver, + model_id=model_id, + task=task, + context=context, + workspace_slug=workspace_slug, + server_env=server_env, + row=row, + repetition=repetition, + is_api_driver=is_api_driver, ) - if context.get("project_name"): - print( - f" orphaned project may remain: {context['project_name']}", - file=sys.stderr, + if agent is not None: + _apply_agent_run( + row, + agent, + model_alias=model_alias, + requested_tier=requested_tier, + model_id=model_id, + external=external, ) - else: - if "bug_type" in task_needs and not context.get("bug_type"): - reason = context.get("bug_type_skip_reason") or "bug_type unavailable" - row.skipped = reason - row.verify_note = reason - print(f" {task['id']} rep={repetition} SKIPPED: {reason}") - else: - agent: TaskResult | None = None - # Agent wrap: API failures and CLI failures are infrastructure. - # Contained CLI stops (timeout / error subtypes) return AgentRun. - try: - agent = await run_agent_task_via_driver( - driver=driver, - model_id=model_id, - task=task, - ctx=context, - workspace_slug=workspace_slug, - optimal_tools=set(task["optimal_tools"]), - alternate_tools=set(task["alternate_tools"]), - server_env=server_env, - ) - except PromptBindError as exc: - # Empty/missing seed IDs in the prompt — not an agent failure. - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "infra_seed" - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", - file=sys.stderr, - ) - agent = None - except Exception as exc: - if is_api_driver: - agent_error_class = "infra_api" - else: - agent_error_class = "infra_cli" - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = agent_error_class - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", - file=sys.stderr, - ) - agent = None - if agent is not None: - row.apply_agent_result(agent) - # Driver-level requested_model is the resolved ID. - # Restore run-level intent and retain both identities. - row.requested_model = model_alias - row.requested_tier = requested_tier - row.resolved_model = model_id - if external: - # Foreign tool names are not comparable to our catalog. - row.alternate_calls = None - row.out_of_set_calls = None - # CLI infra stops: timeout + error subtypes except error_max_turns. - stop_reason = agent.stop_reason - if driver_name.endswith("-cli") and is_infra_cli_stop_reason( - str(stop_reason) if stop_reason is not None else None - ): - row.success = False - row.error_class = "infra_cli" - if stop_reason == "timeout": - row.error = _timeout_error_message(agent) - else: - notes = [note for note in agent.driver_notes if isinstance(note, str)] - detail = "; ".join(notes) if notes else str(stop_reason) - row.error = detail - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", - file=sys.stderr, - ) - else: - verify = task["verify"] - try: - agent_row = agent.to_row() - ok, note = await verify( - plane, - context, - { - "final_text": agent.final_text, - "calls": agent_row["calls"], - }, - ) - row.success = bool(ok) - row.verify_note = note - print(f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}") - except TaskSkipped as skip: - row.skipped = skip.reason - row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") - except Exception as exc: - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "task" - row.verify_note = "" - print( - f" {task['id']} rep={repetition} ERROR[task]: {exc}", - file=sys.stderr, - ) + if not _record_cli_infra_stop( + row, + agent, + task=task, + repetition=repetition, + driver_name=driver_name, + ): + await _verify_task(plane, task, context, agent, row=row, repetition=repetition) except Exception as exc: # Anything outside seed/driver/verify wraps. - row.success = False - row.error = f"{type(exc).__name__}: {exc}" - row.error_class = "task" - row.verify_note = "" - print(f" {task['id']} rep={repetition} ERROR[task]: {exc}", file=sys.stderr) - if context.get("project_name"): - print( - f" orphaned project may remain: {context['project_name']}", - file=sys.stderr, - ) + _record_unexpected(row, exc, task=task, repetition=repetition, context=context) finally: - try: - teardown(plane, context) - except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) - if context.get("project_name"): - print(f" orphaned project: {context['project_name']}", file=sys.stderr) + _remove_fixtures(plane, context) return row diff --git a/tests/test_evals_hardening.py b/tests/test_evals_hardening.py index 6f5c13d5..fbe689a3 100644 --- a/tests/test_evals_hardening.py +++ b/tests/test_evals_hardening.py @@ -223,10 +223,31 @@ def test_live_run_rejects_non_positive_reps(capsys): # --------------------------------------------------------------------------- +def _taxonomy_task( + task_id: str, + verify: Any, + *, + prompt: str = "do {project}", + needs: set[str] | None = None, +) -> dict[str, Any]: + return { + "id": task_id, + "prompt": prompt, + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": {"search_work_items"}, + "optimal_calls": 1, + "needs": set(needs or set()), + "verify": verify, + } + + def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" fake_plane = MagicMock() + driver = MagicMock() + torn: list[dict[str, Any]] = [] monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) def boom_seed(plane, run_id, needs, ctx): @@ -234,7 +255,8 @@ def boom_seed(plane, run_id, needs, ctx): raise HttpError("identifier already taken", 409) monkeypatch.setattr(runner_live, "seed", boom_seed) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) task = { "id": "T1", @@ -265,6 +287,7 @@ def boom_seed(plane, run_id, needs, ctx): assert row["schema_version"] == RESULT_SCHEMA_VERSION assert row["error_class"] == "infra_seed" assert row["success"] is False + assert row["verify_note"] == "" assert "HttpError" in (row["error"] or "") assert "identifier" in (row["error"] or "").lower() assert row["battery"] # fingerprint written @@ -276,6 +299,121 @@ def boom_seed(plane, run_id, needs, ctx): assert meta["schema_version"] == RESULT_SCHEMA_VERSION assert meta["requested_tier"] == "standard" assert meta["resolved_model"] == "sonnet" + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL deadbeef"}] + + +def test_run_live_missing_bug_type_uses_context_skip_reason(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + def seed_without_bug_type(plane, run_id, needs, ctx): + ctx.update( + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "plan:work-item-types-disabled", + } + ) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "BUGTYPE", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + needs={"bug_type"}, + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "plan:work-item-types-disabled" + assert row["verify_note"] == "plan:work-item-types-disabled" + assert row["error"] is None + assert row["error_class"] is None + driver.run_task.assert_not_called() + assert torn == [ + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "plan:work-item-types-disabled", + } + ] + + +def test_run_live_prompt_bind_failure_is_infra_seed(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "PROMPT", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + prompt="use {missing_seed_id} in {project}", + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["verify_note"] == "" + assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] + + +def test_run_live_api_driver_exception_is_infra_api(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.side_effect = RuntimeError("provider unavailable") + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "APIERR", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + ) + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="api", + resolved_model_id="provider-model-id", + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_api" + assert row["verify_note"] == "" + assert row["success"] is False + assert row["error"] == "RuntimeError: provider unavailable" + driver.run_task.assert_called_once() + assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): @@ -504,6 +642,175 @@ async def verify(*a, **k): assert verify_calls == [1] +def test_run_live_verifier_skip_is_not_a_failure(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def skip_verify(plane, ctx, run): + raise TaskSkipped("env:verification-unavailable") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYSKIP", skip_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "env:verification-unavailable" + assert row["verify_note"] == "env:verification-unavailable" + assert row["success"] is False + assert row["error"] is None + assert row["error_class"] is None + assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] + + +def test_run_live_verifier_exception_is_task_error(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def broken_verify(plane, ctx, run): + raise ValueError("verifier broke") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYERR", broken_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is False + assert row["error_class"] == "task" + assert row["error"] == "ValueError: verifier broke" + assert row["verify_note"] == "" + assert row["skipped"] is None + assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] + + +def test_run_live_external_server_nulls_catalog_mispicks(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "search_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "external ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("EXTERNAL", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + server_cmd=["/bin/foreign", "stdio"], + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["server"] == "external" + assert row["alternate_calls"] is None + assert row["out_of_set_calls"] is None + + +def test_run_live_success_keeps_requested_and_resolved_models(tmp_path: Path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "list_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "local ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("SUCCESS", verify_ok)], + model_alias="standard", + resolved_model_id="provider-model-id", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "provider-model-id" + assert row["server"] == "local" + + def test_is_infra_cli_stop_reason_matrix(): assert is_infra_cli_stop_reason("timeout") is True assert is_infra_cli_stop_reason("error_during_execution") is True @@ -1071,6 +1378,7 @@ def test_task_skipped_from_seed_records_a_skip_row(tmp_path: Path, monkeypatch): out = tmp_path / "out.jsonl" driven: list[str] = [] torn: list[Any] = [] + driver = MagicMock() def skip_seed(*_args: Any, **_kwargs: Any) -> None: raise TaskSkipped("env:no-activity-worker") @@ -1081,7 +1389,7 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: monkeypatch.setattr( runner_live, "get_driver", - lambda *a, **k: driven.append("ran") or MagicMock(), + lambda *a, **k: driven.append("ran") or driver, ) tasks = [ @@ -1106,6 +1414,7 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: assert row["error_class"] is None assert row["label"] == "local" assert torn == [1] # teardown still runs + driver.run_task.assert_not_called() # `skipped` is the discriminator, not `success` — a skip must leave the # success denominator empty rather than counting as a failed task. From 6ad1b1c92cd24b5def2e8d63ad7fc2a2d25d208b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 18:56:42 +0530 Subject: [PATCH 23/93] Group the tests the way the harness is grouped The nine test_evals_*.py files were grouped by when they were written. Finding where something was tested meant guessing: test_evals_proxy.py was 2011 lines covering the recording proxy, the CLI driver template and model-tier resolution, and test_evals_hardening.py was 1423 lines importing from eight modules. tests/evals/ now mirrors the harness, so each source module has one obvious place. The autouse credential fixture and the row reader stop being duplicated per file and live in tests/evals/conftest.py. Placement only, proven two ways: pytest collects exactly the same 374 tests before and after, and all 300 test functions are byte-identical to their previous versions apart from one fixture path that had to change because its file moved two directories deeper. The five non-eval test files are untouched. They do carry three ruff findings that a wider lint scope now surfaces; all three already exist on origin/main and are left alone. --- tests/evals/__init__.py | 0 tests/evals/conftest.py | 30 + tests/evals/drivers/__init__.py | 0 .../drivers/test_api_driver.py} | 112 +- tests/evals/drivers/test_cli_driver.py | 748 ++++++++++++ .../drivers/test_vendors.py} | 854 ++++++------- tests/evals/report/__init__.py | 0 tests/evals/report/test_compare.py | 78 ++ tests/evals/report/test_load.py | 109 ++ tests/evals/report/test_summary.py | 142 +++ tests/evals/report/test_table.py | 269 ++++ tests/evals/runner/__init__.py | 0 tests/evals/runner/test_canary.py | 101 ++ .../runner/test_live.py} | 780 ++---------- tests/evals/runner/test_resume.py | 326 +++++ tests/evals/seed/__init__.py | 0 .../seed/test_seed.py} | 530 +++++--- tests/evals/tasks/__init__.py | 0 tests/evals/tasks/test_answers.py | 30 + tests/evals/tasks/test_catalog.py | 263 ++++ tests/evals/tasks/test_debias_verifiers.py | 557 +++++++++ tests/evals/tasks/test_output_contracts.py | 335 +++++ .../tasks/test_verifiers.py} | 33 +- tests/evals/test_cli.py | 143 +++ tests/evals/test_listing.py | 44 + .../test_proxy.py} | 973 +-------------- tests/evals/test_results.py | 359 ++++++ tests/evals/test_token_counting.py | 86 ++ tests/evals/test_tool_names.py | 24 + tests/test_evals_debias_verifiers.py | 1082 ----------------- tests/test_evals_output_contracts.py | 120 -- tests/test_evals_report_ops.py | 780 ------------ 32 files changed, 4449 insertions(+), 4459 deletions(-) create mode 100644 tests/evals/__init__.py create mode 100644 tests/evals/conftest.py create mode 100644 tests/evals/drivers/__init__.py rename tests/{test_evals_api_driver.py => evals/drivers/test_api_driver.py} (91%) create mode 100644 tests/evals/drivers/test_cli_driver.py rename tests/{test_evals_drivers.py => evals/drivers/test_vendors.py} (55%) create mode 100644 tests/evals/report/__init__.py create mode 100644 tests/evals/report/test_compare.py create mode 100644 tests/evals/report/test_load.py create mode 100644 tests/evals/report/test_summary.py create mode 100644 tests/evals/report/test_table.py create mode 100644 tests/evals/runner/__init__.py create mode 100644 tests/evals/runner/test_canary.py rename tests/{test_evals_hardening.py => evals/runner/test_live.py} (51%) create mode 100644 tests/evals/runner/test_resume.py create mode 100644 tests/evals/seed/__init__.py rename tests/{test_evals_catalog.py => evals/seed/test_seed.py} (53%) create mode 100644 tests/evals/tasks/__init__.py create mode 100644 tests/evals/tasks/test_answers.py create mode 100644 tests/evals/tasks/test_catalog.py create mode 100644 tests/evals/tasks/test_debias_verifiers.py create mode 100644 tests/evals/tasks/test_output_contracts.py rename tests/{test_evals_verifiers.py => evals/tasks/test_verifiers.py} (95%) create mode 100644 tests/evals/test_cli.py create mode 100644 tests/evals/test_listing.py rename tests/{test_evals_proxy.py => evals/test_proxy.py} (50%) create mode 100644 tests/evals/test_results.py create mode 100644 tests/evals/test_token_counting.py create mode 100644 tests/evals/test_tool_names.py delete mode 100644 tests/test_evals_debias_verifiers.py delete mode 100644 tests/test_evals_output_contracts.py delete mode 100644 tests/test_evals_report_ops.py diff --git a/tests/evals/__init__.py b/tests/evals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/evals/conftest.py b/tests/evals/conftest.py new file mode 100644 index 00000000..253408dc --- /dev/null +++ b/tests/evals/conftest.py @@ -0,0 +1,30 @@ +"""Shared fixtures and helpers for eval harness tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def _eval_creds(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +def _data_rows(path: Path) -> list[dict]: + """Parse JSONL skipping meta / non-task lines.""" + out: list[dict] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row.get("row_type") == "meta" or row.get("task_id") is None: + continue + out.append(row) + return out diff --git a/tests/evals/drivers/__init__.py b/tests/evals/drivers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_evals_api_driver.py b/tests/evals/drivers/test_api_driver.py similarity index 91% rename from tests/test_evals_api_driver.py rename to tests/evals/drivers/test_api_driver.py index 1a411616..9594e8e7 100644 --- a/tests/test_evals_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -1,4 +1,4 @@ -"""Offline tests for the provider-generic API eval driver.""" +"""Offline eval tests for api driver.""" from __future__ import annotations @@ -26,7 +26,6 @@ resolve_backend_model, unregister_backend, ) -from evals.results import agent_run_to_harness_dict from evals.token_counting import estimate_result_tokens @@ -108,6 +107,26 @@ def run_driver(driver: ApiDriver, *, max_turns: int = 5): ) +class FakeAnthropicMessages: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + +class FakeOpenAICompletions: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + def test_registered_third_party_backend_runs_without_driver_changes(): created: list[FakeBackend] = [] @@ -353,80 +372,6 @@ def test_api_driver_uses_optional_backend_token_counter(): assert run.token_count_failures == 0 -def test_api_driver_maps_every_legacy_row_field(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage=Usage(4, 1), - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="done", - tool_calls=[], - usage=None, - stop_reason=StopReason.END_TURN, - provider_stop_reason="fake_done", - ), - ] - ) - run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) - row = agent_run_to_harness_dict( - run, - optimal={"lookup"}, - alternate=set(), - classify=lambda tool, optimal, alternate: ( - "optimal" if tool in optimal else "alternate" if tool in alternate else "out_of_set" - ), - ) - - required = { - "final_text", - "calls", - "num_calls", - "errored_calls", - "alternate_calls", - "out_of_set_calls", - "total_result_tokens", - "usage_per_iteration", - "cum_input_tokens", - "wall_time_s", - "stop_reason", - "provider_stop_reason", - "hit_max_iterations", - "result_pair_mismatch", - "token_count_failures", - } - assert required <= row.keys() - assert { - "tool", - "class", - "args_chars", - "result_tokens", - "result_chars", - "result_kind", - "is_error", - } <= row["calls"][0].keys() - assert row["calls"][0]["result_chars"] == 5 - assert row["calls"][0]["result_tokens"] == estimate_result_tokens(5) == 2 - assert row["result_tokens_estimated"] is True - assert row["provider"] == "fake" - assert row["model"] == "fake-actual" - assert row["requested_model"] == "fake-requested" - assert row["provider_stop_reason"] == "fake_done" - - -class FakeAnthropicMessages: - def __init__(self, responses: list[dict[str, Any]]) -> None: - self.responses = deque(responses) - self.requests: list[dict[str, Any]] = [] - - def create(self, **kwargs): - self.requests.append(copy.deepcopy(kwargs)) - return self.responses.popleft() - - @pytest.mark.parametrize( ("raw_reason", "expected"), [ @@ -512,16 +457,6 @@ def test_anthropic_backend_translates_tools_turns_and_results(): assert backend.actual_model == "claude-actual" -class FakeOpenAICompletions: - def __init__(self, responses: list[dict[str, Any]]) -> None: - self.responses = deque(responses) - self.requests: list[dict[str, Any]] = [] - - def create(self, **kwargs): - self.requests.append(copy.deepcopy(kwargs)) - return self.responses.popleft() - - @pytest.mark.parametrize( ("raw_reason", "expected"), [ @@ -680,11 +615,6 @@ def test_openai_backend_normalizes_refusal_for_driver_guard(): assert turn.tool_calls == [ToolCall("danger", "write", {})] -# --------------------------------------------------------------------------- -# MCP translation helpers -# --------------------------------------------------------------------------- - - def test_tool_spec_from_mcp_reads_dict_and_object_entries(): from evals.drivers.driver import tool_spec_from_mcp diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py new file mode 100644 index 00000000..dc3e51d6 --- /dev/null +++ b/tests/evals/drivers/test_cli_driver.py @@ -0,0 +1,748 @@ +"""Offline eval tests for cli driver.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +import time +from pathlib import Path + +import pytest + +from evals.drivers import ( + AntigravityCliDriver, + ClaudeCliDriver, + CodexCliDriver, + OpencodeCliDriver, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_wrap_server_command, + run_cli_subprocess, + wait_for_proxy_meta, +) +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # exists but not owned by us + return True + + +REPO = Path(__file__).resolve().parents[3] + + +def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path: Path): + """Timeout kills the whole process group, not just the parent (codex node→native case). + + Sticky CLI: parent spawns a grandchild in the same group that would keep + stdout open if only the parent were killed. Assert the runner returns + quickly and both PIDs are dead. + """ + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_cli.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + # Grandchild stays in the same process group (no start_new_session). + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + # Hold our stdout open forever (simulates grandchild pipe hold). + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired) as ei: + run_cli_subprocess( + [sys.executable, str(script)], + timeout=1.0, + capture_output=True, + text=True, + ) + elapsed = time.monotonic() - t0 + assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" + assert getattr(ei.value, "killed_process_group", False) is True + + # Wait briefly for reaping + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"process group members still alive: {alive}" + + +def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): + """killpg(leader_pid) works after the leader is reaped (no getpgid / no proc.kill fallback). + + Simulates: leader already gone, only grandchild remains in the process group. + """ + import signal + from types import SimpleNamespace + + from evals.drivers import kill_process_group + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_leader.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + leader = subprocess.Popen( + [sys.executable, str(script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2: + break + time.sleep(0.02) + assert len(pids) == 2, pids + leader_pid, child_pid = pids + + # Kill ONLY the leader (not the group) — grandchild survives in the group. + os.kill(leader_pid, signal.SIGKILL) + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + pass + assert not _pid_alive(leader_pid) + assert _pid_alive(child_pid), "precondition: grandchild must still be alive" + + t0 = time.monotonic() + # Direct killpg(leader_pid) — pgid == original leader pid under start_new_session. + ok = kill_process_group(SimpleNamespace(pid=leader_pid)) + assert ok is True + assert time.monotonic() - t0 < 3.0 + + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline and _pid_alive(child_pid): + time.sleep(0.05) + assert not _pid_alive(child_pid), "grandchild survived killpg after leader death" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass + + +def test_run_cli_subprocess_baseexception_kills_group(tmp_path: Path, monkeypatch): + """Non-TimeoutExpired exceptions mid-communicate must still kill the process group.""" + import evals.drivers as drivers_mod + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + real_comm = subprocess.Popen.communicate + calls = {"n": 0} + + def boom_communicate(self, *a, **k): + calls["n"] += 1 + if calls["n"] == 1: + # Wait until pidfile is written so we can assert both die. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: + break + time.sleep(0.02) + raise KeyboardInterrupt("injected mid-communicate") + return real_comm(self, *a, **k) + + monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_cli_subprocess( + [sys.executable, str(script)], + timeout=30.0, + capture_output=True, + text=True, + ) + assert time.monotonic() - t0 < 6.0 + + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2 + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"group survived BaseException path: {alive}" + # silence unused import lint if any + assert drivers_mod.run_cli_subprocess is run_cli_subprocess + + +def test_cli_driver_timeout_notes_process_group_kill(tmp_path: Path): + """ClaudeCliDriver timeout path records timeout_killed_process_group note.""" + script = tmp_path / "slow.py" + script.write_text( + textwrap.dedent( + """ + import time + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. + from evals.drivers import run_cli_subprocess as real_runner + + def short_timeout_runner(cmd, **kwargs): + kwargs = dict(kwargs) + kwargs["timeout"] = 0.5 + # Replace the CLI binary with our sticky sleeper + return real_runner([sys.executable, str(script)], **kwargs) + + driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) + t0 = time.monotonic() + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert time.monotonic() - t0 < 6.0 + assert run.stopped_reason == "timeout" + assert "timeout_killed_process_group" in run.notes + + +def test_old_payload_free_sidecar_still_parses(tmp_path: Path): + path = tmp_path / "old.jsonl" + path.write_text( + json.dumps( + { + "tool": "legacy", + "args": {}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n" + + json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}) + + "\n", + encoding="utf-8", + ) + + calls, status = load_proxy_sidecar(path) + assert status["state"] == "complete" + assert calls[0]["result_chars"] == 17 + assert "result_text" not in calls[0] + + +def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path: Path): + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps( + { + "tool": "find_work_items", + "args": {"q": "x"}, + "is_error": False, + "result_chars": 12, + "duration_ms": 5, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, client, src = apply_proxy_sidecar( + [{"tool": "old", "args": {}, "origin": "plane"}], + [], + side, + notes, + ) + assert src == "proxy" + assert calls[0]["tool"] == "find_work_items" + assert calls[0]["duration_ms"] == 5 + assert any("calls_from_proxy" in n for n in notes) + + +def test_apply_proxy_sidecar_empty_fallback(tmp_path: Path): + side = tmp_path / "empty.jsonl" + side.write_text("", encoding="utf-8") + notes: list[str] = [] + original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] + calls, _client, src = apply_proxy_sidecar(original, [], side, notes) + assert calls is original or calls == original + assert "proxy_sidecar_empty" in notes + assert src != "proxy" or calls == original + + +def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): + clock = {"now": 0.0} + monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) + + class MinimalCliDriver(CliDriver): + name = "minimal-cli" + temp_dir_prefix = "plane-eval-minimal-" + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir, child_env + # Harness-owned setup takes five seconds on the fake clock. The + # persisted wall time must start after this hook returns. + clock["now"] = 5.0 + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del model, max_turns, system, launch + return ["minimal", prompt] + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del proc, task_cwd, max_turns, notes + return CliOutput( + final_text="done", + calls=[ + {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, + {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, + ], + ) + + def write_complete_sidecar(path: Path, tool: str) -> None: + rows = [ + { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + }, + {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + success_driver: MinimalCliDriver + + def success_runner(cmd, **kwargs): + write_complete_sidecar(success_driver.sidecar_path, "proxy_first") + clock["now"] = 7.0 + return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") + + success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) + success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert success.call_source == "proxy" + assert [call["tool"] for call in success.calls] == ["proxy_first"] + assert success.wall_time_s == 2.0 + + timeout_driver: MinimalCliDriver + + def timeout_runner(cmd, **kwargs): + write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") + clock["now"] = 8.0 + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) + + timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) + timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert timed_out.stopped_reason == "timeout" + assert timed_out.call_source == "proxy" + assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] + assert timed_out.wall_time_s == 3.0 + + +def test_proxy_wrap_server_command(): + out = proxy_wrap_server_command( + ["python", "-m", "plane_mcp", "stdio"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="/venv/bin/python", + ) + assert out[:5] == ["/venv/bin/python", "-m", "evals.proxy", "--log", "/tmp/s.jsonl"] + assert out[5] == "--" + assert out[6:] == ["python", "-m", "plane_mcp", "stdio"] + + with_payloads = proxy_wrap_server_command( + ["server"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="python", + record_result_payloads=True, + ) + assert with_payloads[5:7] == ["--record-result-payloads", "--"] + + +def test_load_proxy_sidecar_sorts_by_seq(tmp_path: Path): + p = tmp_path / "s.jsonl" + # Append in reverse response order. + rows = [ + {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, + {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls = load_proxy_sidecar_calls(p) + assert [c["tool"] for c in calls] == ["a", "b"] + + +def test_load_proxy_sidecar_torn_final_line(tmp_path: Path): + p = tmp_path / "s.jsonl" + good = { + "tool": "a", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + # Complete call row + torn final line (no proxy_meta). + p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status["torn_line"] is True + assert status["meta"] is None + assert [c["tool"] for c in calls] == ["a"] + + +def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path: Path): + p = tmp_path / "s.jsonl" + # Incomplete: one proxy call, no meta. + p.write_text( + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in calls] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): + def make_fake(driver_cls: type, bag: dict): + def fake_run(cmd, **kwargs): + bag["cmd"] = cmd + if driver_cls is ClaudeCliDriver and "--mcp-config" in cmd: + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is OpencodeCliDriver: + cwd = kwargs.get("cwd") + if cwd: + cfg = Path(cwd) / "opencode.json" + if cfg.is_file(): + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is AntigravityCliDriver: + env = kwargs.get("env") or {} + home = env.get("HOME") + if home: + for rel in ( + Path(".gemini") / "config" / "mcp_config.json", + Path(".gemini") / "antigravity-cli" / "mcp_config.json", + ): + p = Path(home) / rel + if p.is_file(): + bag.setdefault("cfgs", []).append(json.loads(p.read_text())) + elif driver_cls is CodexCliDriver: + bag["cmd_joined"] = " ".join(cmd) + out = ( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + if driver_cls is ClaudeCliDriver + else "{}" + ) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + return fake_run + + for Driver, bin_key in ( + (ClaudeCliDriver, "claude_bin"), + (CodexCliDriver, "codex_bin"), + (AntigravityCliDriver, "agy_bin"), + (OpencodeCliDriver, "opencode_bin"), + ): + seen: dict = {} + kwargs = { + "runner": make_fake(Driver, seen), + "use_proxy": True, + "record_result_payloads": True, + "python_bin": sys.executable, + "server_command": ["/ext/bin/foreign-mcp", "stdio", "--mode", "candidate"], + } + if Driver is CodexCliDriver: + kwargs["allow_live"] = True + kwargs[bin_key] = "fake-bin" + driver = Driver(**kwargs) + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + blob = json.dumps(seen) + assert "foreign-mcp" in blob or "foreign-mcp" in seen.get("cmd_joined", "") + assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") + + +def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") + + for Driver in (AntigravityCliDriver, OpencodeCliDriver): + d = Driver(runner=fake_run, use_proxy=False, python_bin=sys.executable) + run = d.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.call_source != "proxy" + + +def test_ensure_proxy_pythonpath_injects_repo(): + env = ensure_proxy_pythonpath({}) + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + # Idempotent + env2 = ensure_proxy_pythonpath(env) + assert env2["PYTHONPATH"].count(str(REPO)) == 1 + + +def test_timeout_harvests_sidecar_calls(tmp_path: Path): + """Claude timeout path must include sidecar calls made before the timeout.""" + side_calls = [ + { + "tool": "pre_timeout", + "args": {"a": 1}, + "is_error": False, + "result_chars": 3, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + + def fake_run(cmd, **kwargs): + # Plant a complete sidecar next to the mcp config (temp dir still alive). + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + # Sidecar path is in the same temp dir as mcp.json for Claude. + # Find sidecar from proxy args in mcp config. + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + log_idx = args.index("--log") + 1 + side = Path(args[log_idx]) + side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "pre_timeout" + + +def test_timeout_harvest_waits_for_delayed_meta(tmp_path: Path): + """After CLI kill, harvest must poll until proxy_meta appears (not read early).""" + import threading + import time as time_mod + + call_row = { + "tool": "late_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + meta_row = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": False, + } + seen: dict = {"waited": False} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + # Call row first — no meta yet (simulates proxy still finalizing). + side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") + + def write_meta_later() -> None: + time_mod.sleep(0.45) + with side.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta_row) + "\n") + seen["waited"] = True + + threading.Thread(target=write_meta_later, daemon=True).start() + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + t0 = time_mod.monotonic() + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + elapsed = time_mod.monotonic() - t0 + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "late_meta_tool" + assert seen["waited"] is True + # Must have waited for the delayed meta (~0.45s), not returned instantly. + assert elapsed >= 0.4 + assert "proxy_meta_wait_timeout" not in run.notes + + +def test_wait_for_proxy_meta_unit(tmp_path: Path): + side = tmp_path / "s.jsonl" + side.write_text("", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=0.15, poll_s=0.05) is False + side.write_text(json.dumps({"row_type": "proxy_meta", "pending_left": 0}) + "\n", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=1.0, poll_s=0.05) is True + + +def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): + """If meta never arrives, harvest still returns with incomplete note.""" + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps({"tool": "only", "args": {}, "seq": 1, "is_error": False, "result_chars": 0}) + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, _client, src = harvest_proxy_after_cli_timeout([], [], side, notes, max_wait_s=0.25) + assert "proxy_meta_wait_timeout" in notes + assert len(calls) == 1 + assert src == "proxy" + assert any("incomplete" in n for n in notes) diff --git a/tests/test_evals_drivers.py b/tests/evals/drivers/test_vendors.py similarity index 55% rename from tests/test_evals_drivers.py rename to tests/evals/drivers/test_vendors.py index 030a407e..d690b70d 100644 --- a/tests/test_evals_drivers.py +++ b/tests/evals/drivers/test_vendors.py @@ -1,48 +1,36 @@ -"""Offline tests for eval agent drivers (no real CLI invocations).""" +"""Offline eval tests for vendors.""" from __future__ import annotations import json -import os import subprocess import sys -import textwrap -import time from pathlib import Path from typing import Any import pytest -from evals.cli import parse_args from evals.drivers import ( KNOWN_DRIVERS, + AntigravityCliDriver, ApiDriver, ClaudeCliDriver, CodexCliDriver, + OpencodeCliDriver, get_driver, normalize_claude_usage, parse_claude_json_result, parse_claude_transcript_calls, parse_codex_jsonl_events, - run_cli_subprocess, + prepare_antigravity_fake_home, + write_antigravity_mcp_config, write_claude_mcp_config, + write_opencode_mcp_config, ) -from evals.results import AgentRun, agent_run_to_harness_dict -from evals.runner.live import classify_call, stdio_server_env -from evals.token_counting import estimate_result_tokens from evals.tool_names import ( - is_plane_mcp_tool, - normalize_tool_call, split_plane_and_client_calls, - strip_mcp_prefix, ) -# --------------------------------------------------------------------------- -# Fixtures (constructed — never captured from live CLIs) -# --------------------------------------------------------------------------- - -# Mirrors real claude -p --output-format json (probed): input_tokens is uncached-only; -# mass lives in cache_* + modelUsage. CLAUDE_JSON_RESULT = { "type": "result", "subtype": "success", @@ -80,7 +68,6 @@ }, } -# JSON result that already embeds tool_calls (rare path) CLAUDE_JSON_WITH_CALLS = { **CLAUDE_JSON_RESULT, "tool_calls": [ @@ -100,7 +87,6 @@ } -# Transcript rows (assistant + tool_use) — Claude project JSONL shape def _transcript_lines(*, include_tool_search: bool = False) -> str: content_blocks: list[dict] = [] if include_tool_search: @@ -220,26 +206,37 @@ def _transcript_lines(*, include_tool_search: bool = False) -> str: ] ) +CODEX_V0147_JSONL = "\n".join( + [ + json.dumps( + { + "type": "thread.started", + "thread_id": "019ff6af-69df-7022-b353-322ffe1ececb", + } + ), + json.dumps({"type": "turn.started"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "PING"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 16050, + "cached_input_tokens": 15104, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "reasoning_output_tokens": 0, + }, + } + ), + ] +) -# --------------------------------------------------------------------------- -# strip / parse unit tests -# --------------------------------------------------------------------------- - - -def test_strip_mcp_prefix(): - assert strip_mcp_prefix("mcp__plane__list_work_items") == "list_work_items" - assert strip_mcp_prefix("mcp__plane-mcp-server__find_work_items") == "find_work_items" - assert strip_mcp_prefix("list_work_items") == "list_work_items" - assert strip_mcp_prefix("Bash") == "Bash" - - -def test_is_plane_mcp_tool(): - assert is_plane_mcp_tool("mcp__plane__find_work_items") - assert is_plane_mcp_tool("mcp__plane-foo__x") - assert not is_plane_mcp_tool("ToolSearch") - assert not is_plane_mcp_tool("Bash") - assert not is_plane_mcp_tool("mcp__other__tool") - assert not is_plane_mcp_tool("find_work_items") +REPO = Path(__file__).resolve().parents[3] def test_normalize_claude_usage_real_shape(): @@ -301,38 +298,6 @@ def test_parse_codex_jsonl_events(): assert out["stopped_reason"] == "end_turn" -# Live-captured codex v0.147.0 `codex exec --json` shape (exact four lines). -CODEX_V0147_JSONL = "\n".join( - [ - json.dumps( - { - "type": "thread.started", - "thread_id": "019ff6af-69df-7022-b353-322ffe1ececb", - } - ), - json.dumps({"type": "turn.started"}), - json.dumps( - { - "type": "item.completed", - "item": {"id": "item_0", "type": "agent_message", "text": "PING"}, - } - ), - json.dumps( - { - "type": "turn.completed", - "usage": { - "input_tokens": 16050, - "cached_input_tokens": 15104, - "cache_write_input_tokens": 0, - "output_tokens": 5, - "reasoning_output_tokens": 0, - }, - } - ), - ] -) - - def test_parse_codex_jsonl_events_v0147_schema(): """Parser fixture: exact four-line v0.147 stream (thread_id, PING, usage).""" out = parse_codex_jsonl_events(CODEX_V0147_JSONL) @@ -612,26 +577,6 @@ def fake_run(cmd, **kwargs): assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" -def test_agent_run_dict_keeps_action_arg(): - run = AgentRun( - calls=[ - {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, - {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, - ], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - d = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=lambda t, o, a: "out_of_set", - ) - assert d["calls"][0]["action"] == "create" - assert "action" not in d["calls"][1] - - def test_codex_driver_parses_fake_stdout_no_live(): def fake_run(cmd, **kwargs): assert cmd[0] == "codex" @@ -662,459 +607,346 @@ def test_codex_driver_refuses_live_by_default(): driver.run_task("x", mcp_env={}, model=None, max_turns=1) -def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): - """F1: ToolSearch must not inflate out_of_set or num_calls.""" - run = AgentRun( - calls=[ - normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), - ], - client_tool_calls=[ - normalize_tool_call("ToolSearch", {"query": "work items"}), - ], - final_text="done", - usage={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_cost_usd": 0.29, - "modelUsage": { - "claude-sonnet": { - "inputTokens": 10, - "outputTokens": 865, - "cacheReadInputTokens": 250433, - "cacheCreationInputTokens": 33838, - "costUSD": 0.29, - } - }, - }, - usage_total={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_input_tokens_including_cache": 10 + 250433 + 33838, - "total_cost_usd": 0.29, - "source": "modelUsage", - }, - stopped_reason="end_turn", - usage_scope="run", - call_source="transcript", - hit_max_turns=False, - wall_time_s=1.5, - ) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate={"get_work_item"}, - classify=classify_call, - ) - assert out["num_calls"] == 1 - assert out["out_of_set_calls"] == 0 - assert out["calls"][0]["class"] == "optimal" - assert out["client_tool_call_count"] == 1 - assert out["client_tool_calls"][0]["tool"] == "ToolSearch" - # F2: cum_input_tokens null — not the misleading uncached-only 10 - assert out["cum_input_tokens"] is None - assert out["cum_input_tokens_reason"] - assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 - assert out["usage_per_iteration"] == [] - assert out["calls"][0]["result_tokens"] == 0 - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True - assert "result_tokens_skipped_reason" not in out - - -def test_agent_run_hit_max_maps_to_hit_max_iterations(): - run = AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - hit_max_turns=True, - call_source="json", - ) - out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) - assert out["hit_max_iterations"] is True - assert out["stop_reason"] == "max_turns" - - -def test_agent_run_to_harness_dict_does_not_guess_usage_total(): - """Generic row mapping must not invent usage_total from a vendor usage dict. - - Drivers own normalization (ClaudeCliDriver via normalize_claude_usage, - CodexCliDriver builds its own). Missing usage_total stays None. - """ - run = AgentRun( - calls=[], - final_text="ok", - usage={ - "input_tokens": 5000, - "output_tokens": 200, - # Codex-ish shape — not Claude modelUsage. A Claude rebuild would - # silently produce a wrong / empty total if reintroduced. - "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, - }, - usage_total=None, - stopped_reason="completed", - usage_scope="run", - call_source="stream", - ) - out = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=classify_call, - ) - assert out["usage"] == run.usage - assert out["usage_total"] is None +def test_known_drivers(): + assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} -def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): - class FakeEncoding: - def encode(self, text): - assert text == "serialized workspace result" - return [10, 20, 30] +def test_get_driver_api(): + assert isinstance(get_driver("api"), ApiDriver) + assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) + assert isinstance(get_driver("codex-cli"), CodexCliDriver) - class FakeTiktoken: - @staticmethod - def get_encoding(name): - assert name == "cl100k_base" - return FakeEncoding() - monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len("serialized workspace result"), - "result_text": "serialized workspace result", - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", - ) - - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, +def test_parse_claude_json_preserves_error_subtype(): + out = parse_claude_json_result( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "x", + "session_id": "s", + "num_turns": 1, + } ) + assert out["stopped_reason"] == "error_during_execution" - assert out["calls"][0]["result_tokens"] == 3 - assert out["calls"][0]["result_tokens_estimated"] is False - assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" - assert out["result_tokens_estimated"] is False - assert out["result_tokens_mode"] == "measured" - assert "result_text" not in out["calls"][0] +def test_claude_driver_timeout_returns_agent_run_not_raise(): + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) -def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): - monkeypatch.setitem(sys.modules, "tiktoken", None) - text = "payload without a tokenizer" - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len(text), - "result_text": text, - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=2, + cwd=Path("/tmp"), ) + assert run.stopped_reason == "timeout" + assert run.calls == [] + assert any("timeout after" in n for n in run.notes) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, - ) - assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True +def test_claude_driver_json_parse_failure_raises_for_infra_cli(): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") + driver = ClaudeCliDriver(runner=fake_run) + with pytest.raises(RuntimeError, match="claude cli failed"): + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=Path("/tmp"), + ) -# --------------------------------------------------------------------------- -# Plumbing -# --------------------------------------------------------------------------- +def test_claude_driver_uses_proxy_in_mcp_config(tmp_path: Path): + seen: dict = {} -def test_known_drivers(): - assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + # Leave empty sidecar (proxy not really run under fake runner). + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["command"] == "/venv/bin/python" + assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + assert "plane_mcp" in server["args"] + assert "proxy_sidecar_empty" in run.notes -def test_get_driver_api(): - assert isinstance(get_driver("api"), ApiDriver) - assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) - assert isinstance(get_driver("codex-cli"), CodexCliDriver) +def test_claude_driver_proxy_disabled_no_wrap(tmp_path: Path): + seen: dict = {} + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) -def test_parse_args_accepts_driver(): - a = parse_args(["--driver", "claude-cli", "--dry-run"]) - assert a.driver == "claude-cli" - b = parse_args(["--dry-run"]) - assert b.driver == "api" - assert b.model == "standard" - assert b.provider == "anthropic" - assert b.record_result_payloads is False - c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) - assert c.record_result_payloads is True - - -def test_stdio_env_still_works_for_cli_drivers(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") - monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - env = stdio_server_env() - assert env["PLANE_API_KEY"] == "k" - assert "ANTHROPIC_API_KEY" not in env - - -def _pid_alive(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True # exists but not owned by us - return True - - -def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path: Path): - """Timeout kills the whole process group, not just the parent (codex node→native case). - - Sticky CLI: parent spawns a grandchild in the same group that would keep - stdout open if only the parent were killed. Assert the runner returns - quickly and both PIDs are dead. - """ - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky_cli.py" - script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - pidfile = Path({str(pidfile)!r}) - # Grandchild stays in the same process group (no start_new_session). - child = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(9999)"], - ) - pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") - # Hold our stdout open forever (simulates grandchild pipe hold). - time.sleep(9999) - """ - ), - encoding="utf-8", + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["args"] == ["-m", "plane_mcp", "stdio"] - t0 = time.monotonic() - with pytest.raises(subprocess.TimeoutExpired) as ei: - run_cli_subprocess( - [sys.executable, str(script)], - timeout=1.0, - capture_output=True, - text=True, - ) - elapsed = time.monotonic() - t0 - assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" - assert getattr(ei.value, "killed_process_group", False) is True - - # Wait briefly for reaping - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): - break - time.sleep(0.05) - assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"process group members still alive: {alive}" - - -def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): - """killpg(leader_pid) works after the leader is reaped (no getpgid / no proc.kill fallback). - - Simulates: leader already gone, only grandchild remains in the process group. - """ - import signal - from types import SimpleNamespace - - from evals.drivers import kill_process_group - - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky_leader.py" - script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - pidfile = Path({str(pidfile)!r}) - child = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(9999)"], - ) - pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") - time.sleep(9999) - """ - ), - encoding="utf-8", + +def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + env = kwargs.get("env") or {} + seen["env"] = env + home = env.get("HOME") + if home: + cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" + seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, + model="gemini-2.5", + max_turns=5, + cwd=tmp_path, ) - leader = subprocess.Popen( - [sys.executable, str(script)], - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + assert seen["cmd"][0] == "agy" + assert "-p" in seen["cmd"] + assert "--output-format" in seen["cmd"] + assert "json" in seen["cmd"] + assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] + assert "no_turn_cap" in run.notes + assert seen.get("mcp_cfg") is not None + assert "mcpServers" in seen["mcp_cfg"] + assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) + + +def test_write_antigravity_mcp_config_shape(tmp_path: Path): + p = tmp_path / "mcp_config.json" + write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) + data = json.loads(p.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + assert data["mcpServers"]["plane"]["env"]["A"] == "1" + + +def test_opencode_driver_writes_project_config(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cwd = kwargs.get("cwd") + seen["cwd"] = cwd + cfg = Path(cwd) / "opencode.json" if cwd else None + seen["opencode_cfg"] = json.loads(cfg.read_text()) if cfg and cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + driver = OpencodeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="openai/gpt-test", + max_turns=4, + cwd=tmp_path, ) - try: - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2: - break - time.sleep(0.02) - assert len(pids) == 2, pids - leader_pid, child_pid = pids - - # Kill ONLY the leader (not the group) — grandchild survives in the group. - os.kill(leader_pid, signal.SIGKILL) - try: - leader.wait(timeout=2.0) - except subprocess.TimeoutExpired: - pass - assert not _pid_alive(leader_pid) - assert _pid_alive(child_pid), "precondition: grandchild must still be alive" - - t0 = time.monotonic() - # Direct killpg(leader_pid) — pgid == original leader pid under start_new_session. - ok = kill_process_group(SimpleNamespace(pid=leader_pid)) - assert ok is True - assert time.monotonic() - t0 < 3.0 - - deadline = time.monotonic() + 3.0 - while time.monotonic() < deadline and _pid_alive(child_pid): - time.sleep(0.05) - assert not _pid_alive(child_pid), "grandchild survived killpg after leader death" - finally: - if leader.poll() is None: - try: - os.killpg(leader.pid, signal.SIGKILL) - except Exception: - leader.kill() - try: - leader.wait(timeout=2.0) - except Exception: - pass - - -def test_run_cli_subprocess_baseexception_kills_group(tmp_path: Path, monkeypatch): - """Non-TimeoutExpired exceptions mid-communicate must still kill the process group.""" - import evals.drivers as drivers_mod - - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky.py" - script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - pidfile = Path({str(pidfile)!r}) - child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) - pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") - time.sleep(9999) - """ - ), - encoding="utf-8", + assert seen["cmd"][0] == "opencode" + assert "run" in seen["cmd"] + assert "--format" in seen["cmd"] and "json" in seen["cmd"] + assert "-m" in seen["cmd"] and "openai/gpt-test" in seen["cmd"] + assert "no_turn_cap" in run.notes + data = seen["opencode_cfg"] + assert data is not None + assert data["mcp"]["plane"]["type"] == "local" + assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) + + +def test_write_opencode_mcp_config_shape(tmp_path: Path): + p = tmp_path / "opencode.json" + write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) + data = json.loads(p.read_text()) + assert data["mcp"]["plane"]["command"][0] == "py" + assert data["mcp"]["plane"]["environment"]["K"] == "V" + + +def test_known_drivers_and_get_driver(): + assert "antigravity-cli" in KNOWN_DRIVERS + assert "opencode-cli" in KNOWN_DRIVERS + assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) + assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) + + +def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): + real_home = tmp_path / "real" + cli = real_home / ".gemini" / "antigravity-cli" + cli.mkdir(parents=True) + token_path = cli / "antigravity-oauth-token" + token_path.write_text("secret", encoding="utf-8") + # Snapshot real home before setup — must be byte-identical after. + before = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + + fake = tmp_path / "fake" + prepare_antigravity_fake_home( + fake, + command="python", + args=["-m", "evals.proxy", "--log", "s", "--", "x"], + env={"PLANE_API_KEY": "k"}, + real_home=real_home, ) + p1 = fake / ".gemini" / "config" / "mcp_config.json" + p2 = fake / ".gemini" / "antigravity-cli" / "mcp_config.json" + assert p1.is_file() and p2.is_file() + fake_cli = fake / ".gemini" / "antigravity-cli" + assert fake_cli.is_dir() and not fake_cli.is_symlink() + # Auth artifact is a plain COPY — never a symlink (no write-through path). + token = fake_cli / "antigravity-oauth-token" + assert token.is_file() and not token.is_symlink() + assert token.read_text(encoding="utf-8") == "secret" + # Writing the fake token must not mutate the real one. + token.write_text("mutated", encoding="utf-8") + assert token_path.read_text(encoding="utf-8") == "secret" + # mcp_config is a real file in the fake tree, not inside real home. + assert not (cli / "mcp_config.json").exists() + data = json.loads(p1.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + # Real home byte-for-byte untouched (including oauth token). + after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + assert after == before + + +def test_antigravity_fallback_runner_timeout_harvests(tmp_path: Path): + """TypeError fallback path's TimeoutExpired must still harvest via wait-for-meta.""" + call_row = { + "tool": "g_tool", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + } - real_comm = subprocess.Popen.communicate - calls = {"n": 0} - - def boom_communicate(self, *a, **k): - calls["n"] += 1 - if calls["n"] == 1: - # Wait until pidfile is written so we can assert both die. - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: + def fake_run(cmd, **kwargs): + run_env = kwargs.get("env") or {} + home = run_env.get("HOME") + if home: + # First attempt includes env= — plant sidecar from dual-written mcp config, + # then reject env so the driver retries without it. + for rel in ( + Path(home) / ".gemini" / "config" / "mcp_config.json", + Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + cfg = json.loads(rel.read_text()) + args = cfg["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text( + "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", + encoding="utf-8", + ) break - time.sleep(0.02) - raise KeyboardInterrupt("injected mid-communicate") - return real_comm(self, *a, **k) - - monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) - - t0 = time.monotonic() - with pytest.raises(KeyboardInterrupt): - run_cli_subprocess( - [sys.executable, str(script)], - timeout=30.0, - capture_output=True, - text=True, - ) - assert time.monotonic() - t0 < 6.0 - - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): - break - time.sleep(0.05) - assert len(pids) == 2 - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"group survived BaseException path: {alive}" - # silence unused import lint if any - assert drivers_mod.run_cli_subprocess is run_cli_subprocess - - -def test_cli_driver_timeout_notes_process_group_kill(tmp_path: Path): - """ClaudeCliDriver timeout path records timeout_killed_process_group note.""" - script = tmp_path / "slow.py" - script.write_text( - textwrap.dedent( - """ - import time - time.sleep(9999) - """ - ), - encoding="utf-8", + raise TypeError("runner does not accept env=") + # Fallback call (no env) times out — outer except must still harvest. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "g_tool" - # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. - from evals.drivers import run_cli_subprocess as real_runner - def short_timeout_runner(cmd, **kwargs): - kwargs = dict(kwargs) - kwargs["timeout"] = 0.5 - # Replace the CLI binary with our sticky sleeper - return real_runner([sys.executable, str(script)], **kwargs) +def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path: Path): + seen: dict = {} - driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) - t0 = time.monotonic() - run = driver.run_task( + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( "hi", mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, model="sonnet", max_turns=1, cwd=tmp_path, ) - assert time.monotonic() - t0 < 6.0 - assert run.stopped_reason == "timeout" - assert "timeout_killed_process_group" in run.notes + assert str(REPO) in seen["env"].get("PYTHONPATH", "") diff --git a/tests/evals/report/__init__.py b/tests/evals/report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py new file mode 100644 index 00000000..b246b20f --- /dev/null +++ b/tests/evals/report/test_compare.py @@ -0,0 +1,78 @@ +"""Offline eval tests for compare.""" + +from __future__ import annotations + +import math + +import pytest + +from evals.report import ( + ab_compare, + sign_test_pvalue, +) + + +def test_sign_test_all_positive_hand_computed(): + """n=5 non-zero, all positive → two-sided p = 2 * (1/32) = 0.0625.""" + deltas = [1.0, 2.0, 3.0, 0.5, 4.0] + p = sign_test_pvalue(deltas) + assert p == pytest.approx(2.0 * (1.0 / 32.0)) + assert p == pytest.approx(0.0625) + + +def test_sign_test_four_of_five_hand_computed(): + """n=5, k=4 positive → right tail (C(5,4)+C(5,5))/32 = 6/32; p=2*6/32=0.375.""" + deltas = [1.0, 1.0, 1.0, 1.0, -1.0] + p = sign_test_pvalue(deltas) + right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 + assert p == pytest.approx(2.0 * right) + assert p == pytest.approx(0.375) + + +def test_sign_test_drops_zeros_and_none_when_empty(): + assert sign_test_pvalue([0.0, 0.0]) is None + assert sign_test_pvalue([]) is None + # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 + assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) + + +def test_ab_compare_paired_deltas_and_sign_test(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired + ] + cmp = ab_compare(rows_a, rows_b) + assert cmp["n_paired"] == 2 + deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} + assert deltas["R1"] == -3.0 + assert deltas["R2"] == 1.0 + assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] + assert cmp["sign_test_p"] is not None + assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 + assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 + + +def test_ab_compare_multi_rep_uses_median_successful_call_counts(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, + ] + + cmp = ab_compare(rows_a, rows_b) + + assert cmp["multi_rep"] is True + assert cmp["unstable_a"] == 1 + assert cmp["unstable_b"] == 0 + assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] diff --git a/tests/evals/report/test_load.py b/tests/evals/report/test_load.py new file mode 100644 index 00000000..6c33007e --- /dev/null +++ b/tests/evals/report/test_load.py @@ -0,0 +1,109 @@ +"""Offline eval tests for load.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.report import ( + dedupe_rows_latest, + is_infra_error_row, + load_rows, + summarize, +) + + +def test_is_infra_error_row_covers_infrastructure_prefix(): + assert is_infra_error_row({"error_class": "infra_cli"}) is True + assert is_infra_error_row({"error_class": "task"}) is False + + +def test_load_rows_dedupe_latest_wins(tmp_path: Path): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p) # default dedupe=latest + assert len(loaded) == 1 + assert loaded[0].num_calls == 9 + assert loaded[0].success is False + + +def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p, dedupe="none") + assert len(loaded) == 2 + err = capsys.readouterr().err + assert "duplicate" in err + assert "R1" in err + + +def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): + p = tmp_path / "r.jsonl" + lines = [ + json.dumps( + { + "row_type": "meta", + "run_id": "abc", + "label": "candidate", + "battery": "deadbeef0001", + "model": "sonnet", + "driver": "claude-cli", + "git_sha": "x", + "ts": "t", + } + ), + json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + rows = load_rows(p) + assert len(rows) == 1 + assert rows[0].task_id == "R1" + + +def test_real_historical_rows_parse_and_report_with_backward_defaults(): + fixture = Path(__file__).parents[2] / "fixtures" / "evals_historical_rows.jsonl" + rows = load_rows(fixture) + + assert [row.schema_version for row in rows] == [0, 0] + by_task = {row.task_id: row for row in rows} + battery4 = by_task["L3"] + assert battery4.final_text == "" + assert battery4.result_tokens_estimated is None + assert battery4.alternate_calls is None + assert battery4.calls[0].result_tokens is None + assert battery4.calls[0].action == "create" + + battery5 = by_task["R2"] + assert battery5.final_text.endswith("\n4") + assert battery5.result_tokens_estimated is True + assert [call.result_tokens for call in battery5.calls] == [315, 64] + + summary = summarize(rows) + assert summary.tasks["L3"].success == "1/1" + assert summary.tasks["L3"].med_calls == 1 + assert summary.tasks["L3"].result_tokens_mode == "unavailable" + assert summary.tasks["R2"].success == "1/1" + assert summary.tasks["R2"].med_calls == 2 + assert summary.tasks["R2"].result_tokens_mode == "estimated" + + +def test_dedupe_rows_latest_pure(): + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 5}, + {"task_id": "R2", "rep": 0, "label": "local", "num_calls": 3}, + ] + out = dedupe_rows_latest(rows) + assert len(out) == 2 + by_id = {r.task_id: r for r in out} + assert by_id["R1"].num_calls == 5 + assert by_id["R2"].num_calls == 3 diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py new file mode 100644 index 00000000..a09b3a31 --- /dev/null +++ b/tests/evals/report/test_summary.py @@ -0,0 +1,142 @@ +"""Offline eval tests for summary.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals import report as report_mod +from evals.report import ( + is_infra_error_row, + load_rows, + summarize, + wilson_interval, +) + + +def test_summarize_excludes_infra_errors_from_success(): + rows = [ + {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "HttpError: 409", + "error_class": "infra_seed", + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "timeout after 120s", + "error_class": "infra_cli", + }, + {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, + ] + summary = summarize(rows) + assert summary.infra_errors == 2 + assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows + assert summary.tasks["R1"].k == 1 + assert summary.tasks["R1"].success == "1/2" + assert summary.tasks["R1"].infra_err == 2 + assert is_infra_error_row(rows[1]) is True + assert is_infra_error_row(rows[0]) is False + + +def test_wilson_interval_bounds(): + lo, hi = wilson_interval(5, 10) + assert lo == pytest.approx(0.2366, abs=1e-4) + assert hi == pytest.approx(0.7634, abs=1e-4) + lo0, hi0 = wilson_interval(0, 10) + assert lo0 == 0.0 + assert hi0 == pytest.approx(0.27754, abs=1e-4) + assert wilson_interval(0, 0) == (0.0, 0.0) + + +def test_summarize_aggregate_wilson_and_call_variance(): + rows = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + ] + s = summarize(rows) + assert s.tasks["R1"].n == 3 + assert s.tasks["R1"].k == 2 + assert s.tasks["R1"].calls_min == 2.0 + assert s.tasks["R1"].calls_max == 6.0 + assert s.tasks["R1"].med_calls == 4.0 + assert s.tasks["R1"].unstable is True + assert s.tasks["R2"].unstable is False + assert s.aggregate_k == 3 + assert s.aggregate_n == 4 + assert s.multi_rep is True + assert s.unstable_task_ids == ["R1"] + assert s.unstable_tasks == 1 + assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 + + +def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): + path = tmp_path / "multi.jsonl" + outcomes = { + "R1": [True, True, True], + "R2": [True, False, True], + "R3": [False, False, False], + } + rows = [ + { + "task_id": task_id, + "rep": rep, + "label": "local", + "success": success, + "num_calls": rep + 1, + "calls": [], + } + for task_id, task_outcomes in outcomes.items() + for rep, success in enumerate(task_outcomes) + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + loaded = load_rows(path) + summary = summarize(loaded) + + assert len(loaded) == 9 # distinct rep keys are not deduped away + assert summary.tasks["R1"].success == "3/3" + assert summary.tasks["R1"].unstable is False + assert summary.tasks["R2"].success == "2/3" + assert summary.tasks["R2"].wilson_lo == pytest.approx(0.2077, abs=1e-4) + assert summary.tasks["R2"].wilson_hi == pytest.approx(0.9385, abs=1e-4) + assert summary.tasks["R2"].unstable is True + assert summary.tasks["R3"].success == "0/3" + assert summary.tasks["R3"].unstable is False + assert summary.unstable_task_ids == ["R2"] + + report_mod.print_table(summary, "Summary: multi.jsonl") + output = capsys.readouterr().out + assert "unstable" in output + r2_line = next(line for line in output.splitlines() if line.startswith("R2")) + assert "2/3" in r2_line + assert "[0.21,0.94]" in r2_line + assert "YES" in r2_line + assert "measured noise floor: 1 task flipped at least once" in output + assert "minimum meaningful difference: 2 tasks" in output + + +def test_single_rep_summary_rendering_is_unchanged(capsys): + rows = [{"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 2, "calls": []}] + + report_mod.print_table(summarize(rows), "Summary: sample.jsonl") + + assert capsys.readouterr().out == ( + "Summary: sample.jsonl\n" + "aggregate success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" + "task n success wilson95 med_calls opt IQR mispick err capped h_err i_err " + "med_rtok p95_rtok med_cum_in\n" + "-------------------------------------------------------------------------------------------------------------------------------\n" + "R1 1 1/1 [0.21,1.00] 2.0 1 2.0-2.0 0.0% 0 0 0 0 " + "- - 0\n" + ) diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py new file mode 100644 index 00000000..0b284d6c --- /dev/null +++ b/tests/evals/report/test_table.py @@ -0,0 +1,269 @@ +"""Offline eval tests for table.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from evals import report as report_mod +from evals.report import ( + build_multi_surface_table, + format_surface_cell, + render_multi_surface_table, + summarize, +) + + +def _synth_row( + tid: str, + *, + rep: int = 0, + success: bool = True, + num_calls: int = 2, + alt: int | None = 0, + oos: int | None = 0, + server: str = "local", + skipped: str | None = None, + error: str | None = None, + error_class: str | None = None, + label: str = "local", +) -> dict[str, Any]: + return { + "task_id": tid, + "rep": rep, + "label": label, + "success": success, + "num_calls": num_calls, + "alternate_calls": alt, + "out_of_set_calls": oos, + "server": server, + "skipped": skipped, + "error": error, + "error_class": error_class, + "calls": [], + } + + +def test_print_table_shows_infra_errors(capsys): + summary = summarize( + [ + {"task_id": "R1", "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "error": "seed failed", "error_class": "infra_seed"}, + {"task_id": "R1", "error": "CLI failed", "error_class": "infra_cli"}, + ] + ) + report_mod.print_table(summary, "Summary: test") + out = capsys.readouterr().out + assert "infra errors: 2" in out + assert "i_err" in out + assert "R1" in out + # per-task infra_err value rendered next to h_err + assert " 2" in out # i_err column value + + +def test_report_marks_entirely_estimated_result_token_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + } + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "estimated" + assert summary.tasks["R1"].result_tokens_mode == "estimated" + + report_mod.print_table(summary, "estimated") + output = capsys.readouterr().out + assert "entirely estimated" in output + assert "med_rtok~" in output + assert "~12" in output + + +def test_report_marks_mixed_measured_and_estimated_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], + "result_tokens_estimated": False, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + }, + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "mixed" + assert summary.tasks["R1"].result_tokens_mode == "mixed" + + report_mod.print_table(summary, "mixed") + output = capsys.readouterr().out + assert "mixed measured and estimated" in output + assert "med_rtok*" in output + + +def test_format_surface_cell_variants(): + assert format_surface_cell(None) == "—" + assert format_surface_cell(_synth_row("R1", skipped="nope")) == "skip" + assert format_surface_cell(_synth_row("R1", error="boom")) == "ERR" + assert format_surface_cell(_synth_row("R1", error_class="infra_seed", error="x")) == "ERR" + assert format_surface_cell(_synth_row("R1", success=True, num_calls=3, alt=0, oos=0)) == "✅ 3c" + assert format_surface_cell(_synth_row("R1", success=False, num_calls=4, alt=1, oos=1)) == "❌ 4c/2mp" + # external: no mispick suffix + assert format_surface_cell(_synth_row("R1", server="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" + + +def test_multi_surface_table_snapshot_with_external(): + local = [ + _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), + _synth_row("R2", label="local", success=False, num_calls=2), + ] + candidate = [ + _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), + _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), + ] + external = [ + _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), + _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), + _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), + ] + table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) + assert table["columns"] == ["local", "candidate", "akhil"] + assert "R1" in table["task_ids"] and "R3" in table["task_ids"] + assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" + assert table["cells"]["R1"]["candidate"] == "✅ 2c" + assert table["cells"]["R1"]["akhil"] == "✅ 3c" + assert table["cells"]["R2"]["candidate"] == "skip" + assert table["cells"]["R3"]["akhil"] == "ERR" + + text = render_multi_surface_table(table, markdown=False) + assert "local" in text and "candidate" in text and "akhil" in text + assert "✅ 3c" in text + assert "skip" in text + assert "ERR" in text + assert "infra 1" in text + + md = render_multi_surface_table(table, markdown=True) + assert md.startswith("| task |") + assert "| R1 |" in md + assert "---" in md + assert "**agg**" in md + + # Footer: external mispicks n/a + assert table["footer"]["akhil"]["mispicks"] is None + assert table["footer"]["local"]["mispicks"] == 1 + assert table["footer"]["akhil"]["infra_errors"] == 1 + + +def test_multi_surface_table_aggregates_reps_and_flags_unstable(): + rows = [ + _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), + _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), + _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), + _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), + _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), + _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), + ] + + table = build_multi_surface_table([("local", rows)]) + + assert table["multi_rep"] is True + assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" + assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" + assert table["footer"]["local"]["success"] == 5 + assert table["footer"]["local"]["n"] == 6 + assert table["footer"]["local"]["unstable_tasks"] == 1 + rendered = render_multi_surface_table(table) + assert "measured noise floor: 1 task flipped at least once" in rendered + assert "minimum meaningful difference: 2 tasks" in rendered + + +def test_single_rep_multi_surface_rendering_is_unchanged(): + rows = [_synth_row("R1", label="local", success=True, num_calls=2)] + + rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) + + assert rendered == ( + "task what local \n" + "-------------------------------------------------------\n" + "R1 In project P, what is the curren… ✅ 2c\n" + "-------------------------------------------------------\n" + "local success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" + ) + + +def test_report_main_table_cli(tmp_path: Path, capsys): + f1 = tmp_path / "a.jsonl" + f2 = tmp_path / "b.jsonl" + f1.write_text( + json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", + encoding="utf-8", + ) + rc = report_mod.main(["--table", str(f1), str(f2)]) + assert rc == 0 + out = capsys.readouterr().out + assert "local" in out and "candidate" in out + assert "R1" in out + + +def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path, capsys): + f1 = tmp_path / "old.jsonl" + f2 = tmp_path / "new.jsonl" + f1.write_text( + json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", + encoding="utf-8", + ) + + rc = report_mod.main(["--table", str(f1), str(f2)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "spans battery fingerprints" in captured.err + assert "different task prompts/questions" in captured.err + + +def test_report_main_markdown_flag(tmp_path: Path, capsys): + f1 = tmp_path / "a.jsonl" + f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") + rc = report_mod.main(["--table", "--markdown", str(f1)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.startswith("| task |") + assert "| R1 |" in out + assert "---" in out + + +def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): + p = tmp_path / "d.jsonl" + rows = [ + _synth_row("R1", label="local", num_calls=1, success=True), + {**_synth_row("R1", label="local", num_calls=9, success=False)}, + ] + # Both rows have the same (task_id, rep, label), so latest-wins keeps one. + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + rc = report_mod.main(["--no-dedupe", str(p)]) + assert rc == 0 + # With no-dedupe, both rows enter summarize → n=2 for R1. + # (dedupe default would leave n=1.) + out = capsys.readouterr().out + assert "R1" in out + assert "2/2" in out or "1/2" in out # one success of two diff --git a/tests/evals/runner/__init__.py b/tests/evals/runner/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/evals/runner/test_canary.py b/tests/evals/runner/test_canary.py new file mode 100644 index 00000000..111c3551 --- /dev/null +++ b/tests/evals/runner/test_canary.py @@ -0,0 +1,101 @@ +"""Offline eval tests for canary.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock + +from evals.runner import canary as runner_canary +from evals.runner import ( + run_canary, +) +from evals.tasks.skip import TaskSkipped + + +def test_canary_detects_broken_verifier(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) + + async def always_ok(plane, ctx, run): + return True, "false positive" + + async def correctly_fails(plane, ctx, run): + return False, "empty agent correctly rejected" + + tasks = [ + { + "id": "GOOD", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": correctly_fails, + }, + { + "id": "BAD", + "prompt": "y {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": always_ok, + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 1 + + +def test_canary_passes_when_all_verifiers_reject(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) + + async def reject(plane, ctx, run): + assert run == {"final_text": "", "calls": []} + return False, "no-op rejected" + + tasks = [ + { + "id": "G1", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": reject, + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 0 + + +def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, + "seed", + lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) + tasks = [ + { + "id": "SKIPME", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (False, "unused"), + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 1 diff --git a/tests/test_evals_hardening.py b/tests/evals/runner/test_live.py similarity index 51% rename from tests/test_evals_hardening.py rename to tests/evals/runner/test_live.py index fbe689a3..9f204ba0 100644 --- a/tests/test_evals_hardening.py +++ b/tests/evals/runner/test_live.py @@ -1,4 +1,4 @@ -"""Offline tests for eval harness hardening (taxonomy, resume, seed retry, fingerprint, canary).""" +"""Offline eval tests for live.""" from __future__ import annotations @@ -9,53 +9,50 @@ from typing import Any from unittest.mock import MagicMock -import pytest from plane.errors.errors import HttpError from evals import cli as run_mod -from evals import report as report_mod -from evals import seed as seed_mod -from evals.drivers import ClaudeCliDriver, parse_claude_json_result -from evals.report import is_infra_error_row, load_rows, summarize +from evals.drivers import ( + ClaudeCliDriver, +) +from evals.report import load_rows, summarize from evals.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult -from evals.runner import canary as runner_canary from evals.runner import ( is_infra_cli_stop_reason, - load_resume_skip_keys, - run_canary, run_live, - should_skip_resume_row, ) from evals.runner import live as runner_live -from evals.seed import create_project_with_identifier_retry, is_identifier_collision -from evals.tasks.catalog import battery_fingerprint, task_author +from evals.runner.live import stdio_server_env from evals.tasks.skip import TaskSkipped - -# Pinned hash of the fixed synthetic catalog in test_battery_fingerprint_stable_and_sensitive. -# Recompute only if the serialization format of battery_fingerprint changes deliberately. -PINNED_SYNTHETIC_BATTERY = "eea5abf36382" +from tests.evals.conftest import _data_rows -def _data_rows(path: Path) -> list[dict]: - """Parse JSONL skipping meta / non-task lines.""" - out: list[dict] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - row = json.loads(line) - if row.get("row_type") == "meta" or row.get("task_id") is None: - continue - out.append(row) - return out +def _taxonomy_task( + task_id: str, + verify: Any, + *, + prompt: str = "do {project}", + needs: set[str] | None = None, +) -> dict[str, Any]: + return { + "id": task_id, + "prompt": prompt, + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": {"search_work_items"}, + "optimal_calls": 1, + "needs": set(needs or set()), + "verify": verify, + } -@pytest.fixture(autouse=True) -def _eval_creds(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") +def test_stdio_env_still_works_for_cli_drivers(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) + env = stdio_server_env() + assert env["PLANE_API_KEY"] == "k" + assert "ANTHROPIC_API_KEY" not in env def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): @@ -69,179 +66,11 @@ def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): assert environment["PLANE_BASE_URL"] == "https://api.plane.so" -# --------------------------------------------------------------------------- -# Resume skip decision (pure) -# --------------------------------------------------------------------------- - - -def test_should_skip_resume_row_completed_success(): - assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True - - -def test_should_skip_resume_row_verify_fail_without_error(): - # Completed attempt (agent ran, verify failed) — do not re-run on resume. - assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True - - -def test_should_skip_resume_row_infra_seed_retries(): - assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False - - -def test_should_skip_resume_row_infra_cli_retries(): - assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False - - -def test_should_skip_resume_row_non_null_error_retries(): - assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False - assert should_skip_resume_row({"error": "boom", "error_class": None}) is False - - -def test_load_resume_skip_keys_summary(tmp_path: Path): - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, - {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, - {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local"), ("W1", 0, "local")} - assert n_skip == 2 - assert n_retry == 1 - - -def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path: Path): - """Historical error row whose later row succeeded must not inflate n_retry.""" - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - assert n_retry == 0 - - -def test_load_resume_skip_keys_label_mismatch(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") - with pytest.raises(SystemExit, match="label"): - load_resume_skip_keys(p, label="local") - - -def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "battery": "aaaaaaaaaaaa", - "model": "sonnet", - "driver": "claude-cli", - "error": None, - } - ) - + "\n", - encoding="utf-8", - ) - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") - with pytest.raises(SystemExit, match="driver"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") - # Missing keys on older rows: pass (back-compat) - p2 = tmp_path / "old.jsonl" - p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") - skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") - assert ("R1", 0, "local") in skip - - -def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): - p = tmp_path / "tiered.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "model": "provider-reported-id", - "requested_model": "standard", - "requested_tier": "standard", - "resolved_model": "old-standard-id", - "error": None, - } - ) - + "\n", - encoding="utf-8", - ) - - skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") - assert skip == {("R1", 0, "local")} - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", model="new-standard-id") - - -def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) - + "\n" - + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated - encoding="utf-8", - ) - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - err = capsys.readouterr().err - assert "invalid JSON" in err - - -def test_load_resume_skip_keys_missing_file(tmp_path: Path): - skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") - assert skip == set() and n_skip == 0 and n_retry == 0 - - -def test_parse_args_resume_and_canary(): - a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) - assert a.resume == "evals/output/x.jsonl" - b = run_mod.parse_args(["--canary", "--tasks", "R1"]) - assert b.canary is True - - def test_live_run_rejects_non_positive_reps(capsys): assert run_mod.main(["--tasks", "R1", "--reps", "0"]) == 2 assert "--reps must be at least 1" in capsys.readouterr().err -# --------------------------------------------------------------------------- -# Error taxonomy (seed raise → infra_seed row) -# --------------------------------------------------------------------------- - - -def _taxonomy_task( - task_id: str, - verify: Any, - *, - prompt: str = "do {project}", - needs: set[str] | None = None, -) -> dict[str, Any]: - return { - "id": task_id, - "prompt": prompt, - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": {"search_work_items"}, - "optimal_calls": 1, - "needs": set(needs or set()), - "verify": verify, - } - - def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): out = tmp_path / "rows.jsonl" @@ -820,394 +649,6 @@ def test_is_infra_cli_stop_reason_matrix(): assert is_infra_cli_stop_reason("max_turns") is False -def test_parse_claude_json_preserves_error_subtype(): - out = parse_claude_json_result( - { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": "x", - "session_id": "s", - "num_turns": 1, - } - ) - assert out["stopped_reason"] == "error_during_execution" - - -# --------------------------------------------------------------------------- -# Driver timeout containment -# --------------------------------------------------------------------------- - - -def test_claude_driver_timeout_returns_agent_run_not_raise(): - def fake_run(cmd, **kwargs): - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) - - driver = ClaudeCliDriver(runner=fake_run) - run = driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=2, - cwd=Path("/tmp"), - ) - assert run.stopped_reason == "timeout" - assert run.calls == [] - assert any("timeout after" in n for n in run.notes) - - -def test_claude_driver_json_parse_failure_raises_for_infra_cli(): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") - - driver = ClaudeCliDriver(runner=fake_run) - with pytest.raises(RuntimeError, match="claude cli failed"): - driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=Path("/tmp"), - ) - - -# --------------------------------------------------------------------------- -# Seed identifier retry -# --------------------------------------------------------------------------- - - -def test_create_project_retries_409_then_succeeds(monkeypatch): - attempts: list[str] = [] - - class FakeProjects: - def create(self, *, workspace_slug, data): - ident = data.identifier - attempts.append(ident) - if len(attempts) < 3: - raise HttpError("Project identifier already taken", 409) - return MagicMock(id="proj-ok", identifier=ident) - - plane = MagicMock() - plane.projects = FakeProjects() - - # Force deterministic retries after first collision. - suffixes = iter(["AAAA", "BBBB"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - - project = create_project_with_identifier_retry( - plane, - "ws", - name="EVAL abcd", - identifier_prefix="EV", - initial_suffix="DEAD", - ) - assert project.id == "proj-ok" - assert attempts[0] == "EVDEAD" - assert len(attempts) == 3 - assert attempts[1] != attempts[0] - assert attempts[2] != attempts[1] - assert attempts[1] == "EVAAAA" - assert attempts[2] == "EVBBBB" - - -def test_create_project_raises_after_max_409s(monkeypatch): - attempts: list[str] = [] - - class Always409: - def create(self, *, workspace_slug, data): - attempts.append(data.identifier) - raise HttpError("identifier already taken", 409) - - plane = MagicMock() - plane.projects = Always409() - suffixes = iter(["1111", "2222", "3333", "should-not-use"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( - plane, - "ws", - name="EVAL x", - identifier_prefix="EV", - initial_suffix="0000", - ) - assert ei.value.status_code == 409 - assert len(attempts) == 3 - assert attempts[0] == "EV0000" - assert attempts[1] != attempts[0] - assert attempts[1] == "EV1111" - assert attempts[2] == "EV2222" - - -def test_create_project_non_collision_error_does_not_retry(): - class Fail500: - def create(self, *, workspace_slug, data): - raise HttpError("server error", 500) - - plane = MagicMock() - plane.projects = Fail500() - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( - plane, - "ws", - name="EVAL x", - identifier_prefix="EV", - initial_suffix="0000", - ) - assert ei.value.status_code == 500 - - -def test_identifier_collision_requires_status_and_language(): - assert is_identifier_collision(HttpError("identifier already taken", 409)) is True - assert is_identifier_collision(HttpError("project exists", 400)) is True - # Validation-shaped: mentions identifier but not collision language → no retry - assert is_identifier_collision(HttpError("identifier is required", 400)) is False - assert is_identifier_collision(HttpError("identifier already taken", 500)) is False - - -# --------------------------------------------------------------------------- -# Battery fingerprint + author -# --------------------------------------------------------------------------- - - -def test_task_author_default(): - assert task_author({}) == "claude" - assert task_author({"author": "alice"}) == "alice" - - -def test_battery_fingerprint_stable_and_sensitive(): - t1 = { - "id": "A", - "prompt": "p1 {project}", - "optimal_tools": {"b", "a"}, - "alternate_tools": {"c"}, - "optimal_calls": 2, - } - t2 = { - "id": "B", - "prompt": "p2", - "optimal_tools": {"x"}, - "alternate_tools": set(), - "optimal_calls": 1, - } - # Order of list must not matter (sorted by id). - h1 = battery_fingerprint([t2, t1]) - h2 = battery_fingerprint([t1, t2]) - assert h1 == h2 == PINNED_SYNTHETIC_BATTERY - assert len(h1) == 12 - - t1_edit = {**t1, "prompt": "p1 edited {project}"} - assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY - - # Subset of selected tasks → different fingerprint (documented ceiling). - assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY - - -def test_battery_fingerprint_catalog_is_nonempty(): - from evals.tasks.catalog import TASKS - - fp = battery_fingerprint() - assert len(fp) == 12 - assert battery_fingerprint(list(TASKS)) == fp - - -def test_battery_fingerprint_changes_with_new_debias_tasks(): - """Adding I/L content must change the catalog fingerprint (content hash).""" - from evals.tasks.catalog import TASKS, TASKS_BY_ID - - full = battery_fingerprint() - without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] - assert without_debias, "pre-debias catalog should be non-empty" - reduced = battery_fingerprint(without_debias) - assert reduced != full - # Single new task also moves the hash relative to a reduced set. - assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced - - -# --------------------------------------------------------------------------- -# Report excludes infra_ rows -# --------------------------------------------------------------------------- - - -def test_summarize_excludes_infra_errors_from_success(): - rows = [ - {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "HttpError: 409", - "error_class": "infra_seed", - }, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "timeout after 120s", - "error_class": "infra_cli", - }, - {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, - ] - summary = summarize(rows) - assert summary.infra_errors == 2 - assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows - assert summary.tasks["R1"].k == 1 - assert summary.tasks["R1"].success == "1/2" - assert summary.tasks["R1"].infra_err == 2 - assert is_infra_error_row(rows[1]) is True - assert is_infra_error_row(rows[0]) is False - - -def test_print_table_shows_infra_errors(capsys): - summary = summarize( - [ - {"task_id": "R1", "success": True, "num_calls": 1, "calls": []}, - {"task_id": "R1", "error": "seed failed", "error_class": "infra_seed"}, - {"task_id": "R1", "error": "CLI failed", "error_class": "infra_cli"}, - ] - ) - report_mod.print_table(summary, "Summary: test") - out = capsys.readouterr().out - assert "infra errors: 2" in out - assert "i_err" in out - assert "R1" in out - # per-task infra_err value rendered next to h_err - assert " 2" in out # i_err column value - - -def test_is_infra_error_row_covers_infrastructure_prefix(): - assert is_infra_error_row({"error_class": "infra_cli"}) is True - assert is_infra_error_row({"error_class": "task"}) is False - - -def test_load_rows_dedupe_latest_wins(tmp_path: Path): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p) # default dedupe=latest - assert len(loaded) == 1 - assert loaded[0].num_calls == 9 - assert loaded[0].success is False - - -def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p, dedupe="none") - assert len(loaded) == 2 - err = capsys.readouterr().err - assert "duplicate" in err - assert "R1" in err - - -# --------------------------------------------------------------------------- -# Canary mode -# --------------------------------------------------------------------------- - - -def test_canary_detects_broken_verifier(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - - async def always_ok(plane, ctx, run): - return True, "false positive" - - async def correctly_fails(plane, ctx, run): - return False, "empty agent correctly rejected" - - tasks = [ - { - "id": "GOOD", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": correctly_fails, - }, - { - "id": "BAD", - "prompt": "y {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": always_ok, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 - - -def test_canary_passes_when_all_verifiers_reject(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - - async def reject(plane, ctx, run): - assert run == {"final_text": "", "calls": []} - return False, "no-op rejected" - - tasks = [ - { - "id": "G1", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": reject, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 0 - - -def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, - "seed", - lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - tasks = [ - { - "id": "SKIPME", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (False, "unused"), - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 - - -# --------------------------------------------------------------------------- -# End-to-end resume -# --------------------------------------------------------------------------- - - def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path: Path, monkeypatch): out = tmp_path / "multi.jsonl" fake_plane = MagicMock() @@ -1264,110 +705,6 @@ async def verify_ok(plane, ctx, run): assert all(row["success"] is True for row in rows) -def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): - out = tmp_path / "resume.jsonl" - # Pre-write: completed R1/0 + infra R2/0 (same label/battery/model/driver as this run). - # Battery is computed from the task list below — seed the file after we know it, - # or write rows without battery (back-compat) and only check skip/retry behavior. - prior = [ - { - "task_id": "R1", - "rep": 0, - "label": "local", - "driver": "claude-cli", - "model": "sonnet", - "error": None, - "error_class": None, - "success": True, - }, - { - "task_id": "R2", - "rep": 0, - "label": "local", - "driver": "claude-cli", - "model": "sonnet", - "error": "HttpError: 409", - "error_class": "infra_seed", - "success": False, - }, - ] - out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") - - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - seed_calls: list[str] = [] - - def ok_seed(plane, run_id, needs, ctx): - # Infer task from empty ctx; runner sets project for verify path. - ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) - seed_calls.append(run_id) - - monkeypatch.setattr(runner_live, "seed", ok_seed) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - class OkDriver: - name = "claude-cli" - - def run_task(self, *args, **kwargs): - return AgentRun( - calls=[{"tool": "list_work_items", "args": {}, "origin": "plane"}], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: OkDriver()) - - async def verify_ok(plane, ctx, run): - return True, "ok" - - tasks = [ - { - "id": "R1", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify_ok, - }, - { - "id": "R2", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify_ok, - }, - ] - - rc = asyncio.run( - run_live( - tasks, - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - resume=True, - ) - ) - assert rc == 0 - # Only R2 should have been re-seeded/run (R1 completed → RESUME_SKIP). - assert len(seed_calls) == 1 - data = _data_rows(out) - # prior 2 + 1 new R2 row (meta may also exist if file was empty — it wasn't) - assert len(data) == 3 - new_r2 = data[-1] - assert new_r2["task_id"] == "R2" - assert new_r2["success"] is True - assert new_r2["error_class"] is None - assert new_r2["final_text"] == "done" - - def test_task_skipped_from_seed_records_a_skip_row(tmp_path: Path, monkeypatch): """A fixture that cannot be seeded records a skip — no agent, no crash. @@ -1421,3 +758,60 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: summary = summarize(load_rows(out)) assert "L2" not in summary.tasks assert summary.aggregate_n == 0 + + +def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): + """--server-cmd must not be Claude-only.""" + from evals.runner import live as run_mod + + captured: dict = {} + + def fake_get_driver(name, **kwargs): + captured["name"] = name + captured["kwargs"] = kwargs + + class Dummy: + def run_task(self, *a, **k): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + call_source="json", + ) + + return Dummy() + + monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + import asyncio + + async def _verify(*a, **k): + return False, "n" + + task = { + "id": "T", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": _verify, + } + rc = asyncio.run( + run_mod.run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=tmp_path / "o.jsonl", + driver_name="opencode-cli", + server_cmd=["/bin/foreign", "stdio"], + ) + ) + assert rc == 0 + assert captured["name"] == "opencode-cli" + assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py new file mode 100644 index 00000000..1c60c472 --- /dev/null +++ b/tests/evals/runner/test_resume.py @@ -0,0 +1,326 @@ +"""Offline eval tests for resume.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from evals.report import ( + is_meta_row, +) +from evals.results import AgentRun +from evals.runner import ( + is_meta_or_non_task_row, + load_resume_skip_keys, + make_run_meta_row, + maybe_write_run_meta, + run_live, + should_skip_resume_row, +) +from evals.runner import live as runner_live +from tests.evals.conftest import _data_rows + + +def test_should_skip_resume_row_completed_success(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True + + +def test_should_skip_resume_row_verify_fail_without_error(): + # Completed attempt (agent ran, verify failed) — do not re-run on resume. + assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True + + +def test_should_skip_resume_row_infra_seed_retries(): + assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False + + +def test_should_skip_resume_row_infra_cli_retries(): + assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False + + +def test_should_skip_resume_row_non_null_error_retries(): + assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False + assert should_skip_resume_row({"error": "boom", "error_class": None}) is False + + +def test_load_resume_skip_keys_summary(tmp_path: Path): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local"), ("W1", 0, "local")} + assert n_skip == 2 + assert n_retry == 1 + + +def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path: Path): + """Historical error row whose later row succeeded must not inflate n_retry.""" + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + assert n_retry == 0 + + +def test_load_resume_skip_keys_label_mismatch(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") + with pytest.raises(SystemExit, match="label"): + load_resume_skip_keys(p, label="local") + + +def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "aaaaaaaaaaaa", + "model": "sonnet", + "driver": "claude-cli", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") + with pytest.raises(SystemExit, match="driver"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") + # Missing keys on older rows: pass (back-compat) + p2 = tmp_path / "old.jsonl" + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0, "local") in skip + + +def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): + p = tmp_path / "tiered.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "model": "provider-reported-id", + "requested_model": "standard", + "requested_tier": "standard", + "resolved_model": "old-standard-id", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + + skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") + assert skip == {("R1", 0, "local")} + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", model="new-standard-id") + + +def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + + "\n" + + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + err = capsys.readouterr().err + assert "invalid JSON" in err + + +def test_load_resume_skip_keys_missing_file(tmp_path: Path): + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") + assert skip == set() and n_skip == 0 and n_retry == 0 + + +def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): + out = tmp_path / "resume.jsonl" + # Pre-write: completed R1/0 + infra R2/0 (same label/battery/model/driver as this run). + # Battery is computed from the task list below — seed the file after we know it, + # or write rows without battery (back-compat) and only check skip/retry behavior. + prior = [ + { + "task_id": "R1", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "error": None, + "error_class": None, + "success": True, + }, + { + "task_id": "R2", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "error": "HttpError: 409", + "error_class": "infra_seed", + "success": False, + }, + ] + out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") + + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_calls: list[str] = [] + + def ok_seed(plane, run_id, needs, ctx): + # Infer task from empty ctx; runner sets project for verify path. + ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) + seed_calls.append(run_id) + + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + class OkDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[{"tool": "list_work_items", "args": {}, "origin": "plane"}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: OkDriver()) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + tasks = [ + { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + }, + { + "id": "R2", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + }, + ] + + rc = asyncio.run( + run_live( + tasks, + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + resume=True, + ) + ) + assert rc == 0 + # Only R2 should have been re-seeded/run (R1 completed → RESUME_SKIP). + assert len(seed_calls) == 1 + data = _data_rows(out) + # prior 2 + 1 new R2 row (meta may also exist if file was empty — it wasn't) + assert len(data) == 3 + new_r2 = data[-1] + assert new_r2["task_id"] == "R2" + assert new_r2["success"] is True + assert new_r2["error_class"] is None + assert new_r2["final_text"] == "done" + + +def test_make_run_meta_row_and_write_once(tmp_path: Path): + path = tmp_path / "out.jsonl" + meta = make_run_meta_row( + run_id="rid", + label="candidate", + server="local", + battery="abcd1234ef00", + model="sonnet", + driver="claude-cli", + git_sha="deadbeef", + ts="2026-01-01T00:00:00+00:00", + ) + assert meta["row_type"] == "meta" + assert is_meta_row(meta) + assert is_meta_or_non_task_row(meta) + assert maybe_write_run_meta(path, meta) is True + # Append a data row — a truncating rewrite on the second call would destroy it. + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True}) + "\n") + assert maybe_write_run_meta(path, meta) is False # file non-empty + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["row_type"] == "meta" + assert json.loads(lines[1])["task_id"] == "R1" + + +def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): + p = tmp_path / "out.jsonl" + p.write_text( + "\n".join( + [ + json.dumps( + { + "row_type": "meta", + "label": "candidate", + "battery": "bbbbbbbbbbbb", + "model": "sonnet", + "driver": "claude-cli", + } + ), + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "candidate", + "error": None, + "error_class": None, + "success": True, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys( + p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + ) + assert skip == {("R1", 0, "candidate")} + assert n_skip == 1 and n_retry == 0 + + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") diff --git a/tests/evals/seed/__init__.py b/tests/evals/seed/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_evals_catalog.py b/tests/evals/seed/test_seed.py similarity index 53% rename from tests/test_evals_catalog.py rename to tests/evals/seed/test_seed.py index 6cbe1cf4..efc93137 100644 --- a/tests/test_evals_catalog.py +++ b/tests/evals/seed/test_seed.py @@ -1,146 +1,64 @@ -"""Offline tests for the full eval task catalog + seed plan + verifiers.""" +"""Offline eval tests for seed.""" from __future__ import annotations import inspect +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock import pytest +from plane.errors.errors import HttpError +from evals import cleanup as cleanup_mod from evals import seed as seed_mod -from evals import tasks as tasks_mod -from evals.cli import cmd_dry_run, cmd_list, parse_args -from evals.seed import seed_plan -from evals.tasks.catalog import TASKS, TASKS_BY_ID, get_tasks - -# DESIGN.md catalog ids (stable) + extras added for uncovered tool families. -DESIGN_IDS = { - "R1", - "R2", - "R3", - "R4", - "R5", - "R6", - "W1", - "W2", - "W3", - "W4", - "W5", - "W6", - "W7", - "W8", - "S1", - "S2", - "S3", - "S4", - "C1", - "C2", -} -EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features -# WS3 de-biasing classes -ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} -LONG_TAIL_IDS = {"L1", "L2", "L3", "L4", "L5"} -# Workspace-scoped prompts that omit {project} -NO_PROJECT_PROMPT_IDS = {"C2", "L3", "L4"} -CATALOG_ID_ORDER = ( - "R1", - "R2", - "R3", - "R4", - "R5", - "R6", - "W1", - "W2", - "W3", - "W4", - "W5", - "W6", - "W7", - "W8", - "W9", - "W10", - "S1", - "S2", - "S3", - "S4", - "S5", - "C1", - "C2", - "R7", - "I1", - "I2", - "I3", - "I4", - "I5", - "L1", - "L2", - "L3", - "L4", - "L5", +from evals.seed import ( + create_project_with_identifier_retry, + is_identifier_collision, + seed_plan, +) +from evals.tasks.debias import ( + L3_TAG_VERSION, + L4_PROP_DISPLAY, ) -@pytest.fixture(autouse=True) -def _eval_creds(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - -def test_catalog_includes_design_and_extras(): - ids = {t["id"] for t in TASKS} - assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" - assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" - assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" - assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" - assert len(TASKS) >= 20 - - -def test_catalog_id_order_is_pinned(): - assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER - - -def test_get_tasks_all_and_filter(): - all_t = get_tasks(None) - assert len(all_t) == len(TASKS) - subset = get_tasks(["R1", "W9", "C2"]) - assert [t["id"] for t in subset] == ["R1", "W9", "C2"] - - -def test_get_tasks_unknown_exits(): - with pytest.raises(SystemExit): - get_tasks(["NOPE"]) - - -def test_task_schema_invariants(): - for t in TASKS: - assert t["id"] - assert isinstance(t["tags"], set) - assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS - assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] - assert isinstance(t["alternate_tools"], set) - assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] - assert callable(t["verify"]) - assert isinstance(t.get("needs"), set) - - -def test_debias_tasks_author(): - from evals.tasks.catalog import task_author - - for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: - t = TASKS_BY_ID[tid] - assert task_author(t) == "post-hoc-debias" - - -def test_w6_seeds_an_open_cycle(): - """W6 asks the agent to close Sprint 12, so the seed must leave it open. - - Plane rejects every edit to an ended cycle, so a pre-closed fixture makes the - task unachievable by design. - """ - assert "cycles_open_past" in TASKS_BY_ID["W6"]["needs"] - assert "cycles" in TASKS_BY_ID["W6"]["needs"] +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +class _TeardownPlane: + def __init__(self): + self.deleted: list[tuple[str, str]] = [] + self.releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)]), + delete=lambda **kw: self.deleted.append(("release_tag", kw["tag_id"])), + ), + delete=lambda **kw: self.deleted.append(("release", kw.get("release_id"))), + ) + self.customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace( + id="prop-1", + display_name=L4_PROP_DISPLAY, + name="eval-industry", + ) + ] + ), + delete=lambda **kw: self.deleted.append(("customer_property", kw["property_id"])), + ), + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) + self.projects = SimpleNamespace(delete=lambda **kw: None) + self.workspace_work_item_types = SimpleNamespace(delete=lambda **kw: None) + self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) def test_seed_plan_covers_all_groups(): @@ -176,53 +94,6 @@ def test_seed_plan_empty_needs_only_project(): assert len(lines) == 2 -def test_verifiers_are_async_and_importable(): - modules = { - "R": "read", - "W": "write", - "S": "schema", - "C": "cross", - "I": "debias", - "L": "debias", - } - for t in TASKS: - fn = t["verify"] - assert inspect.iscoroutinefunction(fn), t["id"] - # Callables resolve without NameError - assert fn.__module__ == f"evals.tasks.{modules[t['id'][0]]}" - - -def test_cmd_list_prints_all_task_ids(capsys): - rc = cmd_list() - assert rc == 0 - out = capsys.readouterr().out - for tid in DESIGN_IDS | EXTRA_IDS: - assert tid in out - - -def test_cmd_dry_run_all_tasks(capsys): - rc = cmd_dry_run(list(TASKS)) - assert rc == 0 - out = capsys.readouterr().out - assert "Seed plan:" in out - for tid in ("R1", "W9", "S4", "C2", "R7"): - assert f"=== {tid} ===" in out - - -def test_parse_args_list(): - a = parse_args(["--list", "--label", "candidate-build"]) - assert a.list is True - assert a.label == "candidate-build" - assert parse_args(["--list"]).label == "local" - - -def test_tasks_module_has_no_hardcoded_uuids(): - """Regression: verifiers must resolve expected values at verify time.""" - src = inspect.getsource(tasks_mod) - # Crude: no UUID-shaped literals in tasks module. - assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) - - def test_seed_module_ast_has_all_group_handlers(): """seed() dispatches every documented fixture group.""" src = inspect.getsource(seed_mod.seed) @@ -556,3 +427,304 @@ def create(self, **kw): assert ("features", creates[0]) in enables assert ("update", creates[1]) in enables assert ("features", creates[1]) in enables + + +def test_teardown_deletes_release_tag_and_customer_property(): + from evals.seed import teardown + + plane = _TeardownPlane() + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "project_name": "EVAL x", + "workspace_objects": [ + {"kind": "release_tag", "id": "tag-tracked"}, + {"kind": "customer_property", "id": "prop-tracked"}, + ], + } + teardown(plane, ctx) + kinds = {k for k, _ in plane.deleted} + assert "release_tag" in kinds + assert "customer_property" in kinds + # Tracked ids deleted + assert ("release_tag", "tag-tracked") in plane.deleted + assert ("customer_property", "prop-tracked") in plane.deleted + + +def test_preclean_removes_stale_tag_and_property(): + from evals.seed import _preclean_ws3_workspace_artifacts + + deleted: list[tuple[str, str]] = [] + + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), + delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), + ) + ) + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), + delete=lambda **kw: deleted.append(("prop", kw["property_id"])), + ) + ) + + _preclean_ws3_workspace_artifacts(Plane(), "ws") + assert ("tag", "t-old") in deleted + assert ("prop", "p-old") in deleted + + +def test_preclean_delete_failure_raises_for_infra_seed(): + """Found artifact that cannot be deleted must raise (harness → infra_seed).""" + from evals.seed import _preclean_ws3_workspace_artifacts + + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), + delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), + ) + ) + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) + ) + + with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): + _preclean_ws3_workspace_artifacts(Plane(), "ws") + + +def test_preclean_empty_list_is_silent(): + from evals.seed import _preclean_ws3_workspace_artifacts + + class Plane: + releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + customers = SimpleNamespace(properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + + _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise + + +def test_l2_activity_gate_raises_when_empty(): + """Empty activities list after comments → TaskSkipped env:no-activity-worker.""" + from types import SimpleNamespace + + from evals.seed import R5_TITLE, _gate_activity_worker + from evals.tasks.skip import TaskSkipped + + class Plane: + work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) + + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + with pytest.raises(TaskSkipped, match="env:no-activity-worker"): + _gate_activity_worker(Plane(), "ws", ctx) + + +def test_l2_activity_gate_proceeds_when_nonempty(): + from types import SimpleNamespace + + from evals.seed import R5_TITLE, _gate_activity_worker + + class Plane: + work_items = SimpleNamespace( + activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) + ) + + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + _gate_activity_worker(Plane(), "ws", ctx) # no raise + + +def test_create_project_retries_409_then_succeeds(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + ident = data.identifier + attempts.append(ident) + if len(attempts) < 3: + raise HttpError("Project identifier already taken", 409) + return MagicMock(id="proj-ok", identifier=ident) + + plane = MagicMock() + plane.projects = FakeProjects() + + # Force deterministic retries after first collision. + suffixes = iter(["AAAA", "BBBB"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + project = create_project_with_identifier_retry( + plane, + "ws", + name="EVAL abcd", + identifier_prefix="EV", + initial_suffix="DEAD", + ) + assert project.id == "proj-ok" + assert attempts[0] == "EVDEAD" + assert len(attempts) == 3 + assert attempts[1] != attempts[0] + assert attempts[2] != attempts[1] + assert attempts[1] == "EVAAAA" + assert attempts[2] == "EVBBBB" + + +def test_create_project_raises_after_max_409s(monkeypatch): + attempts: list[str] = [] + + class Always409: + def create(self, *, workspace_slug, data): + attempts.append(data.identifier) + raise HttpError("identifier already taken", 409) + + plane = MagicMock() + plane.projects = Always409() + suffixes = iter(["1111", "2222", "3333", "should-not-use"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 409 + assert len(attempts) == 3 + assert attempts[0] == "EV0000" + assert attempts[1] != attempts[0] + assert attempts[1] == "EV1111" + assert attempts[2] == "EV2222" + + +def test_create_project_non_collision_error_does_not_retry(): + class Fail500: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + plane = MagicMock() + plane.projects = Fail500() + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 500 + + +def test_identifier_collision_requires_status_and_language(): + assert is_identifier_collision(HttpError("identifier already taken", 409)) is True + assert is_identifier_collision(HttpError("project exists", 400)) is True + # Validation-shaped: mentions identifier but not collision language → no retry + assert is_identifier_collision(HttpError("identifier is required", 400)) is False + assert is_identifier_collision(HttpError("identifier already taken", 500)) is False + + +def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): + projects = [ + SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), + SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), + SimpleNamespace(id="p3", name="Production", identifier="PROD"), + ] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + rc = cleanup_mod.main([]) # dry-run + assert rc == 0 + assert delete_calls == [] + out = capsys.readouterr().out + assert "EVAL deadbeef" in out + assert "dry-run" in out + assert "Production" not in out # prefix filter + + +def test_cleanup_yes_deletes(monkeypatch, capsys): + projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + rc = cleanup_mod.main(["--yes"]) + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0]["project_id"] == "p1" + + +def test_list_projects_with_prefix_filters(): + projects = [ + SimpleNamespace(id="1", name="EVAL a"), + SimpleNamespace(id="2", name="Other"), + SimpleNamespace(id="3", name="EVAL b"), + SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " + ] + calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + calls.append({"workspace_slug": workspace_slug, "params": params}) + assert params is not None + assert params.per_page == 100 + # SDK always populates next_cursor even on last page. + return SimpleNamespace( + results=projects, + next_page_results=False, + next_cursor="100:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "3"] + assert len(calls) == 1 # one page only — no infinite loop on next_cursor + assert calls[0]["params"].cursor is None + + +def test_list_projects_two_page_pagination(): + page1 = [SimpleNamespace(id="1", name="EVAL one")] + page2 = [SimpleNamespace(id="2", name="EVAL two")] + seen_cursors: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + seen_cursors.append(getattr(params, "cursor", None)) + if params.cursor is None: + return SimpleNamespace( + results=page1, + next_page_results=True, + next_cursor="100:0:0", + ) + assert params.cursor == "100:0:0" + return SimpleNamespace( + results=page2, + next_page_results=False, + next_cursor="200:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "2"] + assert seen_cursors == [None, "100:0:0"] diff --git a/tests/evals/tasks/__init__.py b/tests/evals/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/evals/tasks/test_answers.py b/tests/evals/tasks/test_answers.py new file mode 100644 index 00000000..0d2bcb05 --- /dev/null +++ b/tests/evals/tasks/test_answers.py @@ -0,0 +1,30 @@ +"""Offline eval tests for answers.""" + +from __future__ import annotations + + +def test_reports_contract_int_unit(): + """Direct unit cases for the contract helper.""" + from evals.tasks.answers import reports_contract_int + + assert reports_contract_int("count: 3", 3) is True + assert reports_contract_int("count: 2", 3) is False + assert reports_contract_int("-3", 3) is False + assert reports_contract_int("count: -3", 3) is False + assert reports_contract_int("0", 0) is True + assert reports_contract_int("Some prose only", 0) is False + assert reports_contract_int("preamble\ncount: 0\n", 0) is True + # Last contract line wins + assert reports_contract_int("count: 9\ncount: 3", 3) is True + assert reports_contract_int("count: 9\ncount: 3", 9) is False + + +def test_exact_line_contract_helpers_unit(): + from evals.tasks.answers import contract_values, reports_contract_value, reports_contract_values + + text = "prose mentions state Done\nSTATE: In Progress\nitem: B\nitem: A" + assert contract_values(text, "state") == ["In Progress"] + assert reports_contract_value(text, "state", "In Progress") is True + assert reports_contract_value("- state: In Progress", "state", "In Progress") is False + assert reports_contract_values(text, "item", ["A", "B"]) is True + assert reports_contract_values("item: A\nitem: A", "item", ["A"]) is False diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py new file mode 100644 index 00000000..9e541a28 --- /dev/null +++ b/tests/evals/tasks/test_catalog.py @@ -0,0 +1,263 @@ +"""Offline eval tests for catalog.""" + +from __future__ import annotations + +import inspect + +import pytest + +from evals import tasks as tasks_mod +from evals.tasks.catalog import TASKS, TASKS_BY_ID, battery_fingerprint, get_tasks, task_author +from evals.tasks.debias import ( + I1_TITLE, +) + +DESIGN_IDS = { + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "S1", + "S2", + "S3", + "S4", + "C1", + "C2", +} + +EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features + +ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} + +LONG_TAIL_IDS = {"L1", "L2", "L3", "L4", "L5"} + +NO_PROJECT_PROMPT_IDS = {"C2", "L3", "L4"} + +CATALOG_ID_ORDER = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) + +PINNED_SYNTHETIC_BATTERY = "eea5abf36382" + + +def test_catalog_includes_design_and_extras(): + ids = {t["id"] for t in TASKS} + assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" + assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" + assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" + assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" + assert len(TASKS) >= 20 + + +def test_catalog_id_order_is_pinned(): + assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER + + +def test_get_tasks_all_and_filter(): + all_t = get_tasks(None) + assert len(all_t) == len(TASKS) + subset = get_tasks(["R1", "W9", "C2"]) + assert [t["id"] for t in subset] == ["R1", "W9", "C2"] + + +def test_get_tasks_unknown_exits(): + with pytest.raises(SystemExit): + get_tasks(["NOPE"]) + + +def test_task_schema_invariants(): + for t in TASKS: + assert t["id"] + assert isinstance(t["tags"], set) + assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS + assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] + assert isinstance(t["alternate_tools"], set) + assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] + assert callable(t["verify"]) + assert isinstance(t.get("needs"), set) + + +def test_debias_tasks_author(): + from evals.tasks.catalog import task_author + + for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: + t = TASKS_BY_ID[tid] + assert task_author(t) == "post-hoc-debias" + + +def test_w6_seeds_an_open_cycle(): + """W6 asks the agent to close Sprint 12, so the seed must leave it open. + + Plane rejects every edit to an ended cycle, so a pre-closed fixture makes the + task unachievable by design. + """ + assert "cycles_open_past" in TASKS_BY_ID["W6"]["needs"] + assert "cycles" in TASKS_BY_ID["W6"]["needs"] + + +def test_verifiers_are_async_and_importable(): + modules = { + "R": "read", + "W": "write", + "S": "schema", + "C": "cross", + "I": "debias", + "L": "debias", + } + for t in TASKS: + fn = t["verify"] + assert inspect.iscoroutinefunction(fn), t["id"] + # Callables resolve without NameError + assert fn.__module__ == f"evals.tasks.{modules[t['id'][0]]}" + + +def test_tasks_module_has_no_hardcoded_uuids(): + """Regression: verifiers must resolve expected values at verify time.""" + src = inspect.getsource(tasks_mod) + # Crude: no UUID-shaped literals in tasks module. + assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) + + +def test_prompt_bind_strict_empty_raises(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import PromptBindError, format_task_prompt + + t = TASKS_BY_ID["I1"] + with pytest.raises(PromptBindError): + format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) + + +def test_prompt_bind_strict_exception_raises(): + from evals.tasks.prompts import PromptBindError, format_task_prompt + + def boom(_ctx): + raise RuntimeError("seed broken") + + task = { + "id": "X", + "prompt": "do {work_item_id}", + "prompt_bind": boom, + } + with pytest.raises(PromptBindError, match="prompt_bind failed"): + format_task_prompt(task, {"project_name": "P"}, strict=True) + + +def test_prompt_bind_dry_run_markers(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) + assert "" in text + assert "EVAL x" in text + + +def test_prompt_bind_strict_success(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt( + t, + {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, + strict=True, + ) + assert "uuid-abc" in text + assert "<" not in text + + +def test_task_author_default(): + assert task_author({}) == "claude" + assert task_author({"author": "alice"}) == "alice" + + +def test_battery_fingerprint_stable_and_sensitive(): + t1 = { + "id": "A", + "prompt": "p1 {project}", + "optimal_tools": {"b", "a"}, + "alternate_tools": {"c"}, + "optimal_calls": 2, + } + t2 = { + "id": "B", + "prompt": "p2", + "optimal_tools": {"x"}, + "alternate_tools": set(), + "optimal_calls": 1, + } + # Order of list must not matter (sorted by id). + h1 = battery_fingerprint([t2, t1]) + h2 = battery_fingerprint([t1, t2]) + assert h1 == h2 == PINNED_SYNTHETIC_BATTERY + assert len(h1) == 12 + + t1_edit = {**t1, "prompt": "p1 edited {project}"} + assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY + + # Subset of selected tasks → different fingerprint (documented ceiling). + assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY + + +def test_battery_fingerprint_catalog_is_nonempty(): + from evals.tasks.catalog import TASKS + + fp = battery_fingerprint() + assert len(fp) == 12 + assert battery_fingerprint(list(TASKS)) == fp + + +def test_battery_fingerprint_changes_with_new_debias_tasks(): + """Adding I/L content must change the catalog fingerprint (content hash).""" + from evals.tasks.catalog import TASKS, TASKS_BY_ID + + full = battery_fingerprint() + without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] + assert without_debias, "pre-debias catalog should be non-empty" + reduced = battery_fingerprint(without_debias) + assert reduced != full + # Single new task also moves the hash relative to a reduced set. + assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py new file mode 100644 index 00000000..1a913b8d --- /dev/null +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -0,0 +1,557 @@ +"""Offline eval tests for debias verifiers.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from evals.seed import W2_TITLE +from evals.tasks.debias import ( + I1_TITLE, + I3_TITLE, + I4_TITLE, + L1_TITLE, + L2_TITLE, + L3_TAG_VERSION, + L4_PROP_DISPLAY, + L4_PROP_VALUE, + L5_TITLE, + verify_i1, + verify_i2, + verify_i3, + verify_i4, + verify_i5, + verify_l1, + verify_l2, + verify_l3, + verify_l4, + verify_l5, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str = "") -> dict[str, Any]: + return {"final_text": text, "calls": []} + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +class _WIRetrievePlane: + """work_items.retrieve + list by name; optional labels expand.""" + + def __init__( + self, + *, + by_id: dict[str, Any], + by_name: dict[str, str] | None = None, + states: list[Any] | None = None, + ): + self._by_id = by_id + self._by_name = by_name or {} + self._states = states or [] + self.work_items = SimpleNamespace( + list=self._list, + retrieve=self._retrieve, + ) + self.states = SimpleNamespace(list=lambda **kw: _Page(self._states)) + + def _list(self, **kw): + # Minimal name filter support used by _find_item_by_name. + params = kw.get("params") + name = None + if params is not None: + name = getattr(params, "name", None) or (params.get("name") if isinstance(params, dict) else None) + if name and name in self._by_name: + wid = self._by_name[name] + row = self._by_id.get(wid) or _item(wid, name) + return _Page([row]) + return _Page([]) + + def _retrieve(self, **kw): + wid = str(kw["work_item_id"]) + if wid not in self._by_id: + raise LookupError(wid) + return self._by_id[wid] + + +class _I3Plane: + def __init__(self, cycle_item_ids: list[str]): + self.cycles = SimpleNamespace(list_work_items=lambda **kw: _Page([_item(i, f"n-{i}") for i in cycle_item_ids])) + + +class _L1Plane: + def __init__(self, durations: list[int], summary_ids: list[str] | None = None): + self.work_items = SimpleNamespace( + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + rows = [SimpleNamespace(work_item_id=i, duration=90) for i in (summary_ids or [])] + self.projects = SimpleNamespace(get_worklog_summary=lambda **kw: rows) + + +class _L2Plane: + def __init__(self, n_activities: int): + acts = [SimpleNamespace(id=f"a{i}", verb="updated") for i in range(n_activities)] + self.work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: _Page(acts))) + + +class _L3Plane: + def __init__(self, versions: list[str]): + tags = [SimpleNamespace(id=f"t-{v}", version=v) for v in versions] + self.releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page(tags))) + + +class _L4Plane: + def __init__(self, *, props: list[Any], values: dict[str, list[str]]): + self.customers = SimpleNamespace( + properties=SimpleNamespace(list=lambda **kw: _Page(props)), + property_values=SimpleNamespace(list=lambda **kw: values), + ) + + +class _L5Plane: + def __init__(self, n: int): + rows = [SimpleNamespace(id=f"att-{i}") for i in range(n)] + self.work_items = SimpleNamespace(attachments=SimpleNamespace(list=lambda **kw: _Page(rows))) + + +def test_i1_untouched_urgent_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} + ok, note = await verify_i1(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i1_wrong_item_high_target_still_urgent_fails(): + async def _go(): + # Right value on the wrong item; target remains urgent. + plane = _WIRetrievePlane( + by_id={ + "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), + "wi-other": SimpleNamespace(id="wi-other", priority="high"), + } + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} + ok, note = await verify_i1(plane, ctx, _run()) + assert ok is False, note + assert "urgent" in note or "high" in note + + return asyncio.run(_go()) + + +def test_i2_untouched_empty_final_text_fails(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[st], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i2_wrong_state_name_in_text_fails(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[ + st, + SimpleNamespace(id="st-done", name="Done", group="completed"), + ], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("Done")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i2_exact_state_contract_passes(): + async def _go(): + st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") + plane = _WIRetrievePlane( + by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, + states=[st], + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ok, note = await verify_i2(plane, ctx, _run("state: Backlog")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_i3_untouched_not_on_cycle_fails(): + async def _go(): + plane = _I3Plane(["other-1", "other-2"]) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i3_wrong_item_on_cycle_target_missing_fails(): + async def _go(): + plane = _I3Plane(["wrong-item"]) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i4_untouched_no_label_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[])}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i4_wrong_label_attached_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[SimpleNamespace(id="lab-auth")])}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i5_untouched_none_priority_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="none")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_i5_wrong_value_high_fails(): + async def _go(): + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="high")}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, note + assert "high" in note + + return asyncio.run(_go()) + + +def test_l1_untouched_no_worklog_fails(): + async def _go(): + plane = _L1Plane([]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l1_wrong_duration_120_fails(): + async def _go(): + plane = _L1Plane([120], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("Logged 120 minutes; summary ok.")) + assert ok is False, note + assert "90" in note + + return asyncio.run(_go()) + + +def test_l1_empty_summary_with_90m_log_fails(): + """Reviewer counterexample: 90m log present but final text empty → fail.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("")) + assert ok is False, note + assert "logged-minutes" in note.lower() + + return asyncio.run(_go()) + + +def test_l1_one_hundred_ninety_minutes_fails(): + """Reviewer counterexample: English 'ninety' must not satisfy numeric duration.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1( + plane, + ctx, + _run("Logged one hundred ninety minutes. Project summary looks fine."), + ) + assert ok is False, note + assert "duration" in note.lower() or "90" in note or "1.5" in note + + return asyncio.run(_go()) + + +def test_l1_prose_with_correct_facts_but_without_contract_fails(): + """Correct facts in prose do not satisfy the explicit output contract.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("Logged 1.5 hours total.")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l1_ninety_minutes_of_work_fails_by_design(): + """Calibration: prose without contract lines fails by design.""" + + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1(plane, ctx, _run("90 minutes of work")) + assert ok is False, note + assert "logged-minutes" in note.lower() + + return asyncio.run(_go()) + + +def test_l1_exact_duration_and_summary_contract_passes(): + async def _go(): + plane = _L1Plane([90], summary_ids=["wi-l1"]) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ok, note = await verify_l1( + plane, + ctx, + _run("logged-minutes: 90\nsummary-work-item-id: wi-l1"), + ) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l2_untouched_empty_final_text_fails(): + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l2_contract_count_three_passes(): + """Contract line 'count: 3' truth=3 passes.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("Saw some history.\ncount: 3")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l2_contract_count_two_fails_truth_three(): + """Contract 'count: 2' truth=3 fails.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("count: 2")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l2_negative_contract_and_bare_fail_truth_three(): + """'-3' and 'count: -3' fail truth=3 (signed equality).""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok1, _ = await verify_l2(plane, ctx, _run("-3")) + ok2, _ = await verify_l2(plane, ctx, _run("count: -3")) + assert ok1 is False + assert ok2 is False + + return asyncio.run(_go()) + + +def test_l2_prose_only_without_contract_fails_by_design(): + """By design: prose without 'count: N' (or bare int) fails — format is part of the task.""" + + async def _go(): + plane = _L2Plane(3) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ok, note = await verify_l2(plane, ctx, _run("There are 3 activities and some comment phrases.")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l3_untouched_no_tag_fails(): + async def _go(): + plane = _L3Plane([]) + ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l3_wrong_version_tag_fails(): + async def _go(): + plane = _L3Plane(["v0.0.1", "other-rc"]) + ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) + assert ok is False, note + assert L3_TAG_VERSION in note + + return asyncio.run(_go()) + + +def test_l4_untouched_no_property_fails(): + async def _go(): + plane = _L4Plane(props=[], values={}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l4_right_property_wrong_value_fails(): + async def _go(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + plane = _L4Plane(props=[prop], values={"prop-1": ["Startup"]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + assert L4_PROP_VALUE in note or "Startup" in note or "lack" in note + + return asyncio.run(_go()) + + +def test_l4_industry_url_type_with_enterprise_fails(): + """Reviewer counterexample: name contains Industry, URL type, value Enterprise → fail.""" + + async def _go(): + prop = SimpleNamespace( + id="prop-url", + display_name="Industry", # substring / wrong exact name + name="industry", + property_type="URL", + ) + plane = _L4Plane(props=[prop], values={"prop-url": [L4_PROP_VALUE]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l4_exact_text_enterprise_passes(): + async def _go(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + plane = _L4Plane(props=[prop], values={"prop-1": [L4_PROP_VALUE]}) + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} + ok, note = await verify_l4(plane, ctx, _run()) + assert ok is True, note + assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) + + return asyncio.run(_go()) + + +def test_l5_untouched_empty_final_text_fails(): + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l5_bare_zero_passes(): + """Fallback: whole-answer bare '0' still passes truth=0.""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("0")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l5_multiline_ending_count_zero_passes(): + """Multi-line answer ending with 'count: 0' passes truth=0.""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5( + plane, + ctx, + _run("No files on this work item.\ncount: 0"), + ) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_l5_prose_only_without_contract_fails_by_design(): + """By design: prose without contract line fails (format instruction is part of the task).""" + + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("There are 0 attachments.")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_l5_wrong_contract_count_fails(): + async def _go(): + plane = _L5Plane(0) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ok, note = await verify_l5(plane, ctx, _run("count: 10")) + assert ok is False, note + + return asyncio.run(_go()) diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py new file mode 100644 index 00000000..08996b4a --- /dev/null +++ b/tests/evals/tasks/test_output_contracts.py @@ -0,0 +1,335 @@ +"""Offline eval tests for structural output contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, W2_TITLE, W8_TITLE +from evals.tasks.cross import verify_c2 +from evals.tasks.read import verify_r1, verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.write import verify_w2, verify_w4, verify_w8 + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str = "") -> dict[str, Any]: + return {"final_text": text, "calls": []} + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +class _R1Plane: + def __init__(self, state_name: str): + st = SimpleNamespace(id="st-1", name=state_name, group="started") + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("r1", R1_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="r1", name=R1_TITLE, state=st), + ) + self.states = SimpleNamespace( + list=lambda **kw: _Page( + [ + st, + SimpleNamespace(id="st-2", name="Done", group="completed"), + SimpleNamespace(id="st-3", name="Backlog", group="unstarted"), + ] + ) + ) + + +class _W2Plane: + def __init__(self, group: str, name: str): + st = SimpleNamespace(id="st", name=name, group=group) + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w2", W2_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="w2", state=st), + ) + self.states = SimpleNamespace(list=lambda **kw: _Page([st])) + + +class _W4Plane: + def __init__(self, name: str): + self.labels = SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(id=kw["label_id"], name=name), + list=lambda **kw: _Page([SimpleNamespace(id="triage-id", name=name)]), + ) + + +class _W8Plane: + def __init__(self, durations: list[int]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w8", W8_TITLE)]), + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + + +def test_r2_written_number_prose_fails_and_count_contract_passes(): + async def _go(): + state = SimpleNamespace(id="started", name="Started", group="started") + items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] + plane = SimpleNamespace( + states=SimpleNamespace(list=lambda **kwargs: _Page([state])), + work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), + ) + ctx = {"workspace_slug": "ws", "project_id": "project"} + + prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) + contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) + + assert prose_ok is False + assert contract_ok is True, note + + return asyncio.run(_go()) + + +def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): + async def _go(): + overdue = "Session cookie not rotated after login" + ctx = { + "items": {R1_TITLE: "item-1", overdue: "item-2"}, + "r4_active_item_ids": ["item-1", "item-2"], + "r4_overdue_title": overdue, + } + text = f"cycle: {CYCLE_CURRENT}\nitem: {R1_TITLE}\nitem: {overdue}\noverdue: {overdue}" + + ok, note = await verify_r4(object(), ctx, _run(text)) + keyword_only_ok, _ = await verify_r4(object(), ctx, _run(f"cycle: {CYCLE_CURRENT}\noverdue")) + + assert ok is True, note + assert keyword_only_ok is False + + return asyncio.run(_go()) + + +def test_r5_exact_comment_lines_pass_but_free_prose_does_not(): + async def _go(): + ctx = {"r5_comment_phrases": list(R5_COMMENT_PHRASES)} + contract = "\n".join(f"comment: {phrase}" for phrase in reversed(R5_COMMENT_PHRASES)) + prose = f"The discussion covered {R5_COMMENT_PHRASES[0]} and {R5_COMMENT_PHRASES[1]}." + + contract_ok, note = await verify_r5(object(), ctx, _run(contract)) + prose_ok, _ = await verify_r5(object(), ctx, _run(prose)) + + assert contract_ok is True, note + assert prose_ok is False + + return asyncio.run(_go()) + + +def test_r6_exact_project_contract_passes_and_shorthand_fails(): + async def _go(): + expected = "EVAL deadbeef B" + ctx = {"r6_more_bugs_project": expected} + + exact_ok, note = await verify_r6(object(), ctx, _run(f"project: {expected}")) + shorthand_ok, _ = await verify_r6(object(), ctx, _run("The B project has more bugs.")) + + assert exact_ok is True, note + assert shorthand_ok is False + + return asyncio.run(_go()) + + +def test_r7_transition_contract_is_structural(): + async def _go(): + states = [ + SimpleNamespace(name="Backlog"), + SimpleNamespace(name="In Progress"), + SimpleNamespace(name="Done"), + ] + plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(states))) + ctx = {"workspace_slug": "ws", "project_id": "project"} + + exact_ok, note = await verify_r7(plane, ctx, _run("transition: Done")) + prose_ok, _ = await verify_r7(plane, ctx, _run("It can move to Done.")) + + assert exact_ok is True, note + assert prose_ok is False + + return asyncio.run(_go()) + + +def test_c2_correct_changelog_prose_without_contract_fails(): + async def _go(): + changelog = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." + prose = "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff." + + ok, _ = await verify_c2(object(), {"release_changelog_text": changelog}, _run(prose)) + + assert ok is False + + return asyncio.run(_go()) + + +def test_existing_r1_untouched_empty_text_fails(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_r1_wrong_state_in_text_fails(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("Done")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_r1_exact_state_contract_passes(): + async def _go(): + plane = _R1Plane("In Progress") + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], + } + ok, note = await verify_r1(plane, ctx, _run("state: In Progress")) + assert ok is True, note + + return asyncio.run(_go()) + + +def test_existing_r2_wrong_count_in_text_fails(): + async def _go(): + # verify_r2 counts open urgent via SDK; text must match that count. + from evals.tasks.read import verify_r2 as _vr2 + + class Plane: + def __init__(self): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page( + [ + _item("1", "a", priority="urgent", state=SimpleNamespace(group="started")), + _item("2", "b", priority="urgent", state=SimpleNamespace(group="started")), + _item("3", "c", priority="urgent", state=SimpleNamespace(group="started")), + _item("4", "d", priority="urgent", state=SimpleNamespace(group="started")), + ] + ) + ) + self.states = SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) + ) + + # If verifier only checks text against live count, empty/wrong text fails. + ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w2_untouched_not_done_fails(): + async def _go(): + plane = _W2Plane("started", "In Progress") + ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w2_wrong_cancelled_group_fails(): + async def _go(): + plane = _W2Plane("cancelled", "Cancelled") + ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w4_untouched_still_triage_fails(): + async def _go(): + plane = _W4Plane("triage") + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w4_wrong_name_needs_review_fails(): + async def _go(): + plane = _W4Plane("needs-review") + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(plane, ctx, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w8_untouched_no_log_fails(): + async def _go(): + plane = _W8Plane([]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_w8_wrong_duration_fails(): + async def _go(): + plane = _W8Plane([60]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_c2_untouched_empty_text_fails(): + async def _go(): + ctx = {"release_changelog_text": "Changelog entry one: OAuth login hardening."} + ok, note = await verify_c2(object(), ctx, _run("")) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_c2_wrong_release_name_fails(): + async def _go(): + ok, note = await verify_c2( + object(), + {"release_changelog_text": "Changelog entry one: OAuth login hardening."}, + _run("Release 9.9.9 shipped nothing useful."), + ) + assert ok is False, note + + return asyncio.run(_go()) + + +def test_existing_c2_exact_release_and_shipped_contract_passes(): + async def _go(): + ok, note = await verify_c2( + object(), + { + "release_changelog_text": ( + "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." + ) + }, + _run("release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff"), + ) + assert ok is True, note + + return asyncio.run(_go()) diff --git a/tests/test_evals_verifiers.py b/tests/evals/tasks/test_verifiers.py similarity index 95% rename from tests/test_evals_verifiers.py rename to tests/evals/tasks/test_verifiers.py index 8b53038e..51592d28 100644 --- a/tests/test_evals_verifiers.py +++ b/tests/evals/tasks/test_verifiers.py @@ -1,9 +1,4 @@ -"""Adversarial offline tests for eval verifiers (false-PASS / false-FAIL regressions). - -Each test constructs a minimal fake plane client + ctx that embodies the bug -scenario and asserts the fixed verifier now returns the correct outcome. -No network; no live Plane. -""" +"""Offline eval tests for R, W, S, and C verifiers.""" from __future__ import annotations @@ -12,7 +7,6 @@ from types import SimpleNamespace from typing import Any -import pytest from plane.errors.errors import HttpError from evals.seed import ( @@ -31,10 +25,6 @@ from evals.tasks.schema import verify_s3, verify_s5 from evals.tasks.write import verify_w3, verify_w4, verify_w5, verify_w6, verify_w7, verify_w8 -# --------------------------------------------------------------------------- -# Tiny helpers -# --------------------------------------------------------------------------- - class _Page: def __init__(self, results: list[Any] | None = None, next_page_results: bool = False): @@ -55,17 +45,6 @@ def _run() -> dict[str, Any]: return {"final_text": "", "calls": []} -@pytest.fixture(autouse=True) -def _no_redis(monkeypatch): - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - -# --------------------------------------------------------------------------- -# F1 W7 — reverse blocked_by must NOT pass -# --------------------------------------------------------------------------- - - class _DepsDump: def __init__(self, data: dict): self._data = data @@ -584,11 +563,6 @@ def test_f8_seed_r3_due_date_function_matches(): assert due == sun -# --------------------------------------------------------------------------- -# Minor W8 — exactly 120, not >= 120 -# --------------------------------------------------------------------------- - - class _W8Plane: def __init__(self, durations: list[int]): self.work_items = SimpleNamespace( @@ -652,11 +626,6 @@ async def _go(): return asyncio.run(_go()) -# --------------------------------------------------------------------------- -# S5 — both cycle_view and is_time_tracking_enabled required -# --------------------------------------------------------------------------- - - class _S5Plane: def __init__( self, diff --git a/tests/evals/test_cli.py b/tests/evals/test_cli.py new file mode 100644 index 00000000..87b29564 --- /dev/null +++ b/tests/evals/test_cli.py @@ -0,0 +1,143 @@ +"""Offline eval tests for cli.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from evals import cli as run_mod +from evals.cli import cmd_dry_run, cmd_list, parse_args, resolve_model_for_driver +from evals.cli import main as eval_main +from evals.tasks.catalog import TASKS + +DESIGN_IDS = { + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "S1", + "S2", + "S3", + "S4", + "C1", + "C2", +} + +EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features + + +def test_cmd_list_prints_all_task_ids(capsys): + rc = cmd_list() + assert rc == 0 + out = capsys.readouterr().out + for tid in DESIGN_IDS | EXTRA_IDS: + assert tid in out + + +def test_cmd_dry_run_all_tasks(capsys): + rc = cmd_dry_run(list(TASKS)) + assert rc == 0 + out = capsys.readouterr().out + assert "Seed plan:" in out + for tid in ("R1", "W9", "S4", "C2", "R7"): + assert f"=== {tid} ===" in out + + +def test_parse_args_list(): + a = parse_args(["--list", "--label", "candidate-build"]) + assert a.list is True + assert a.label == "candidate-build" + assert parse_args(["--list"]).label == "local" + + +def test_parse_args_accepts_driver(): + a = parse_args(["--driver", "claude-cli", "--dry-run"]) + assert a.driver == "claude-cli" + b = parse_args(["--dry-run"]) + assert b.driver == "api" + assert b.model == "standard" + assert b.provider == "anthropic" + assert b.record_result_payloads is False + c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) + assert c.record_result_payloads is True + + +def test_parse_args_resume_and_canary(): + a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) + assert a.resume == "evals/output/x.jsonl" + b = run_mod.parse_args(["--canary", "--tasks", "R1"]) + assert b.canary is True + + +def test_model_tiers_resolve_per_driver_and_provider(): + assert resolve_model_for_driver("api", "standard", provider="anthropic") == "claude-sonnet-5" + assert resolve_model_for_driver("api", "fast", provider="anthropic") == "claude-haiku-4-5" + assert resolve_model_for_driver("api", "standard", provider="openai") == "gpt-5.6-sol" + assert resolve_model_for_driver("api", "fast", provider="openai") == "gpt-5.6-luna" + assert resolve_model_for_driver("claude-cli", "standard") == "sonnet" + assert resolve_model_for_driver("claude-cli", "fast") == "haiku" + assert resolve_model_for_driver("codex-cli", "standard") == "gpt-5.6-sol" + assert resolve_model_for_driver("codex-cli", "fast") == "gpt-5.6-luna" + assert resolve_model_for_driver("antigravity-cli", "standard") == "gemini-3.6-flash-high" + assert resolve_model_for_driver("antigravity-cli", "fast") == "gemini-3.6-flash-low" + + +@pytest.mark.parametrize( + ("driver", "model"), + [ + ("api", "claude-opus-5"), + ("api", "sonnet"), + ("claude-cli", "sonnet"), + ("codex-cli", "sonnet"), + ("antigravity-cli", "gemini-3.1-pro-high"), + ("opencode-cli", "haiku"), + ], +) +def test_non_tier_model_strings_pass_through_unchanged(driver, model): + assert resolve_model_for_driver(driver, model) == model + + +def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): + with pytest.raises(ValueError, match=r"opencode models"): + resolve_model_for_driver("opencode-cli", "standard") + + +def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path: Path, capsys): + out = tmp_path / "must-not-exist.jsonl" + + rc = eval_main( + [ + "--driver", + "opencode-cli", + "--model", + "standard", + "--tasks", + "R1", + "--out", + str(out), + ] + ) + + assert rc == 2 + assert "explicit provider/model ID" in capsys.readouterr().err + assert out.exists() is False + + +def test_tier_mapping_is_scoped_to_cli_provider(): + with pytest.raises(ValueError, match=r"codex-cli.*anthropic.*explicit model ID"): + resolve_model_for_driver("codex-cli", "standard", provider="anthropic") + + +def test_qualified_model_id_passes_through_unchanged(): + assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" diff --git a/tests/evals/test_listing.py b/tests/evals/test_listing.py new file mode 100644 index 00000000..6786ec42 --- /dev/null +++ b/tests/evals/test_listing.py @@ -0,0 +1,44 @@ +"""Offline eval tests for listing.""" + +from __future__ import annotations + +from evals.listing import count_tool_tokens, tool_payload_model_facing, tool_payload_wire + + +def test_count_tool_tokens_fake_list(): + class T: + def __init__(self, name, desc, inp, out=None): + self.name = name + self.description = desc + self.inputSchema = inp + self.outputSchema = out + + tools = [ + T("alpha", "short", {"type": "object"}), + T( + "beta", + "longer description here", + {"type": "object", "properties": {"x": {"type": "string"}}}, + out={"type": "object"}, + ), + ] + # Fake encode: 1 token per character (deterministic, no tiktoken needed). + encode = lambda s: list(s) # noqa: E731 + rows, total_wire, total_model = count_tool_tokens(tools, encode=encode) + assert len(rows) == 2 + assert total_wire == sum(r.wire_tokens for r in rows) + assert total_model == sum(r.model_facing_tokens for r in rows) + # Tool with outputSchema has wire > model-facing. + beta = next(r for r in rows if r.name == "beta") + assert beta.has_output_schema is True + assert beta.wire_tokens > beta.model_facing_tokens + alpha = next(r for r in rows if r.name == "alpha") + assert alpha.has_output_schema is False + assert alpha.wire_tokens == alpha.model_facing_tokens + # Sorted by wire desc + assert rows[0].wire_tokens >= rows[1].wire_tokens + + wire = tool_payload_wire(tools[1]) + assert "output_schema" in wire + model = tool_payload_model_facing(tools[1]) + assert "output_schema" not in model diff --git a/tests/test_evals_proxy.py b/tests/evals/test_proxy.py similarity index 50% rename from tests/test_evals_proxy.py rename to tests/evals/test_proxy.py index 2e2b3ef6..97d7e63e 100644 --- a/tests/test_evals_proxy.py +++ b/tests/evals/test_proxy.py @@ -1,4 +1,4 @@ -"""Offline tests for the MCP recording proxy and proxy-first drivers.""" +"""Offline eval tests for proxy.""" from __future__ import annotations @@ -10,27 +10,12 @@ import pytest -from evals.cli import main as eval_main -from evals.cli import resolve_model_for_driver from evals.drivers import ( - KNOWN_DRIVERS, - AntigravityCliDriver, - ClaudeCliDriver, - CodexCliDriver, - OpencodeCliDriver, apply_proxy_sidecar, ensure_proxy_pythonpath, - get_driver, - harvest_proxy_after_cli_timeout, load_proxy_sidecar, load_proxy_sidecar_calls, - prepare_antigravity_fake_home, - proxy_wrap_server_command, - wait_for_proxy_meta, - write_antigravity_mcp_config, - write_opencode_mcp_config, ) -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -41,21 +26,8 @@ write_all_fd, ) from evals.proxy import main as proxy_main -from evals.results import AgentRun, agent_run_to_harness_dict -from evals.token_counting import estimate_result_tokens -REPO = Path(__file__).resolve().parent.parent - - -@pytest.fixture(autouse=True) -def _no_redis(monkeypatch): - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - -# --------------------------------------------------------------------------- -# Fake MCP server script (real subprocess, no network) -# --------------------------------------------------------------------------- +REPO = Path(__file__).resolve().parents[2] FAKE_SERVER = textwrap.dedent( r""" @@ -128,11 +100,6 @@ def _write_fake_server(path: Path) -> Path: return path -# --------------------------------------------------------------------------- -# Proxy round-trip (real subprocess) -# --------------------------------------------------------------------------- - - def test_proxy_records_tools_call_and_exit_code(tmp_path: Path): server = _write_fake_server(tmp_path / "fake_server.py") sidecar = tmp_path / "side.jsonl" @@ -302,31 +269,6 @@ def test_sidecar_result_payload_round_trips_only_when_enabled(tmp_path: Path): assert calls[0]["result_chars"] == len(expected_text) -def test_old_payload_free_sidecar_still_parses(tmp_path: Path): - path = tmp_path / "old.jsonl" - path.write_text( - json.dumps( - { - "tool": "legacy", - "args": {}, - "is_error": False, - "result_chars": 17, - "duration_ms": 1, - "seq": 1, - } - ) - + "\n" - + json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}) - + "\n", - encoding="utf-8", - ) - - calls, status = load_proxy_sidecar(path) - assert status["state"] == "complete" - assert calls[0]["result_chars"] == 17 - assert "result_text" not in calls[0] - - def test_append_after_finalize_is_dropped(tmp_path: Path): """Once write_meta seals the sidecar, further row appends no-op (meta stays last).""" rec = SidecarRecorder(tmp_path / "fin.jsonl") @@ -417,373 +359,11 @@ def test_reap_timeout_floor_when_deadline_exhausted(): assert reap_timeout(future, floor=0.1) >= 4.0 -# --------------------------------------------------------------------------- -# Driver integration: sidecar replaces CLI calls -# --------------------------------------------------------------------------- - - -def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path: Path): - side = tmp_path / "s.jsonl" - side.write_text( - json.dumps( - { - "tool": "find_work_items", - "args": {"q": "x"}, - "is_error": False, - "result_chars": 12, - "duration_ms": 5, - "seq": 1, - } - ) - + "\n", - encoding="utf-8", - ) - notes: list[str] = [] - calls, client, src = apply_proxy_sidecar( - [{"tool": "old", "args": {}, "origin": "plane"}], - [], - side, - notes, - ) - assert src == "proxy" - assert calls[0]["tool"] == "find_work_items" - assert calls[0]["duration_ms"] == 5 - assert any("calls_from_proxy" in n for n in notes) - - -def test_apply_proxy_sidecar_empty_fallback(tmp_path: Path): - side = tmp_path / "empty.jsonl" - side.write_text("", encoding="utf-8") - notes: list[str] = [] - original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] - calls, _client, src = apply_proxy_sidecar(original, [], side, notes) - assert calls is original or calls == original - assert "proxy_sidecar_empty" in notes - assert src != "proxy" or calls == original - - -def test_claude_driver_uses_proxy_in_mcp_config(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - # Leave empty sidecar (proxy not really run under fake runner). - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=3, - cwd=tmp_path, - ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["command"] == "/venv/bin/python" - assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] - assert "--" in server["args"] - assert "plane_mcp" in server["args"] - assert "proxy_sidecar_empty" in run.notes - - -def test_claude_driver_proxy_disabled_no_wrap(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") - driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["args"] == ["-m", "plane_mcp", "stdio"] - - -def test_agent_run_to_harness_propagates_proxy_fields(): - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {"q": "a"}, - "origin": "plane", - "is_error": True, - "result_chars": 99, - "duration_ms": 42, - } - ], - final_text="x", - usage=None, - stopped_reason="end_turn", - call_source="proxy", - usage_scope="run", - ) - d = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=lambda t, o, a: "optimal", - ) - assert d["calls"][0]["is_error"] is True - assert d["calls"][0]["result_chars"] == 99 - assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) - assert d["calls"][0]["result_tokens_estimated"] is True - assert d["result_tokens_estimated"] is True - assert d["calls"][0]["duration_ms"] == 42 - assert d["errored_calls"] == 1 - - -# --------------------------------------------------------------------------- -# Antigravity / OpenCode adapters (arg construction only) -# --------------------------------------------------------------------------- - - -def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): - clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) - - class MinimalCliDriver(CliDriver): - name = "minimal-cli" - temp_dir_prefix = "plane-eval-minimal-" - - def write_mcp_config( - self, - temp_dir: Path, - *, - task_cwd: Path, - server_command: list[str], - child_env: dict[str, str], - ) -> CliLaunch: - del temp_dir, child_env - # Harness-owned setup takes five seconds on the fake clock. The - # persisted wall time must start after this hook returns. - clock["now"] = 5.0 - self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) - return CliLaunch(cwd=task_cwd) - - def build_command( - self, - prompt: str, - *, - model: str | None, - max_turns: int, - system: str | None, - launch: CliLaunch, - ) -> list[str]: - del model, max_turns, system, launch - return ["minimal", prompt] - - def parse_output( - self, - proc: subprocess.CompletedProcess[str], - *, - task_cwd: Path, - max_turns: int, - notes: list[str], - ) -> CliOutput: - del proc, task_cwd, max_turns, notes - return CliOutput( - final_text="done", - calls=[ - {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, - {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, - ], - ) - - def write_complete_sidecar(path: Path, tool: str) -> None: - rows = [ - { - "tool": tool, - "args": {}, - "is_error": False, - "result_chars": 2, - "duration_ms": 1, - "seq": 1, - }, - {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, - ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") - - success_driver: MinimalCliDriver - - def success_runner(cmd, **kwargs): - write_complete_sidecar(success_driver.sidecar_path, "proxy_first") - clock["now"] = 7.0 - return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") - - success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) - success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert success.call_source == "proxy" - assert [call["tool"] for call in success.calls] == ["proxy_first"] - assert success.wall_time_s == 2.0 - - timeout_driver: MinimalCliDriver - - def timeout_runner(cmd, **kwargs): - write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") - clock["now"] = 8.0 - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) - - timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) - timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert timed_out.stopped_reason == "timeout" - assert timed_out.call_source == "proxy" - assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] - assert timed_out.wall_time_s == 3.0 - - -def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - env = kwargs.get("env") or {} - seen["env"] = env - home = env.get("HOME") - if home: - cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" - seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None - return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "do it", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, - model="gemini-2.5", - max_turns=5, - cwd=tmp_path, - ) - assert seen["cmd"][0] == "agy" - assert "-p" in seen["cmd"] - assert "--output-format" in seen["cmd"] - assert "json" in seen["cmd"] - assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] - assert "no_turn_cap" in run.notes - assert seen.get("mcp_cfg") is not None - assert "mcpServers" in seen["mcp_cfg"] - assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) - - -def test_write_antigravity_mcp_config_shape(tmp_path: Path): - p = tmp_path / "mcp_config.json" - write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) - data = json.loads(p.read_text()) - assert data["mcpServers"]["plane"]["command"] == "python" - assert data["mcpServers"]["plane"]["env"]["A"] == "1" - - -def test_opencode_driver_writes_project_config(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - cwd = kwargs.get("cwd") - seen["cwd"] = cwd - cfg = Path(cwd) / "opencode.json" if cwd else None - seen["opencode_cfg"] = json.loads(cfg.read_text()) if cfg and cfg.is_file() else None - return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") - - driver = OpencodeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="openai/gpt-test", - max_turns=4, - cwd=tmp_path, - ) - assert seen["cmd"][0] == "opencode" - assert "run" in seen["cmd"] - assert "--format" in seen["cmd"] and "json" in seen["cmd"] - assert "-m" in seen["cmd"] and "openai/gpt-test" in seen["cmd"] - assert "no_turn_cap" in run.notes - data = seen["opencode_cfg"] - assert data is not None - assert data["mcp"]["plane"]["type"] == "local" - assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) - - -def test_write_opencode_mcp_config_shape(tmp_path: Path): - p = tmp_path / "opencode.json" - write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) - data = json.loads(p.read_text()) - assert data["mcp"]["plane"]["command"][0] == "py" - assert data["mcp"]["plane"]["environment"]["K"] == "V" - - -def test_known_drivers_and_get_driver(): - assert "antigravity-cli" in KNOWN_DRIVERS - assert "opencode-cli" in KNOWN_DRIVERS - assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) - assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) - - -def test_proxy_wrap_server_command(): - out = proxy_wrap_server_command( - ["python", "-m", "plane_mcp", "stdio"], - sidecar_path=Path("/tmp/s.jsonl"), - python_bin="/venv/bin/python", - ) - assert out[:5] == ["/venv/bin/python", "-m", "evals.proxy", "--log", "/tmp/s.jsonl"] - assert out[5] == "--" - assert out[6:] == ["python", "-m", "plane_mcp", "stdio"] - - with_payloads = proxy_wrap_server_command( - ["server"], - sidecar_path=Path("/tmp/s.jsonl"), - python_bin="python", - record_result_payloads=True, - ) - assert with_payloads[5:7] == ["--record-result-payloads", "--"] - - def test_proxy_main_requires_command(): with pytest.raises(SystemExit): proxy_main(["--log", "/tmp/x.jsonl"]) -# --------------------------------------------------------------------------- -# Review-fix coverage -# --------------------------------------------------------------------------- - - def test_server_initiated_request_does_not_pop_pending(tmp_path: Path): """Server message with method+id must not complete a tools/call pending slot.""" rec = SidecarRecorder(tmp_path / "s.jsonl") @@ -849,75 +429,6 @@ def test_write_all_fd_loops_on_short_writes(tmp_path: Path): assert got == payload -def test_load_proxy_sidecar_sorts_by_seq(tmp_path: Path): - p = tmp_path / "s.jsonl" - # Append in reverse response order. - rows = [ - {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, - {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, - { - "row_type": "proxy_meta", - "relayed_lines": 2, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - }, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - calls = load_proxy_sidecar_calls(p) - assert [c["tool"] for c in calls] == ["a", "b"] - - -def test_load_proxy_sidecar_torn_final_line(tmp_path: Path): - p = tmp_path / "s.jsonl" - good = { - "tool": "a", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - # Complete call row + torn final line (no proxy_meta). - p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") - calls, status = load_proxy_sidecar(p) - assert status["state"] == "incomplete" - assert status["torn_line"] is True - assert status["meta"] is None - assert [c["tool"] for c in calls] == ["a"] - - -def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path: Path): - p = tmp_path / "s.jsonl" - # Incomplete: one proxy call, no meta. - p.write_text( - json.dumps( - { - "tool": "from_proxy", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - ) - + "\n", - encoding="utf-8", - ) - cli = [ - {"tool": "c1", "args": {}, "origin": "plane"}, - {"tool": "c2", "args": {}, "origin": "plane"}, - ] - notes: list[str] = [] - calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) - assert src != "proxy" - assert [c["tool"] for c in calls] == ["c1", "c2"] - assert any("proxy_sidecar_incomplete" in n for n in notes) - assert any("deferred_to_cli" in n for n in notes) - - def test_proxy_exits_when_child_dies_first(tmp_path: Path): """Child exits while parent stdin is still open — proxy must not hang.""" server = tmp_path / "die_soon.py" @@ -1044,206 +555,6 @@ def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path: Path): assert calls[0]["tool"] == "ping" -def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): - def make_fake(driver_cls: type, bag: dict): - def fake_run(cmd, **kwargs): - bag["cmd"] = cmd - if driver_cls is ClaudeCliDriver and "--mcp-config" in cmd: - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - bag["cfg"] = json.loads(cfg.read_text()) - elif driver_cls is OpencodeCliDriver: - cwd = kwargs.get("cwd") - if cwd: - cfg = Path(cwd) / "opencode.json" - if cfg.is_file(): - bag["cfg"] = json.loads(cfg.read_text()) - elif driver_cls is AntigravityCliDriver: - env = kwargs.get("env") or {} - home = env.get("HOME") - if home: - for rel in ( - Path(".gemini") / "config" / "mcp_config.json", - Path(".gemini") / "antigravity-cli" / "mcp_config.json", - ): - p = Path(home) / rel - if p.is_file(): - bag.setdefault("cfgs", []).append(json.loads(p.read_text())) - elif driver_cls is CodexCliDriver: - bag["cmd_joined"] = " ".join(cmd) - out = ( - json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ) - if driver_cls is ClaudeCliDriver - else "{}" - ) - return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") - - return fake_run - - for Driver, bin_key in ( - (ClaudeCliDriver, "claude_bin"), - (CodexCliDriver, "codex_bin"), - (AntigravityCliDriver, "agy_bin"), - (OpencodeCliDriver, "opencode_bin"), - ): - seen: dict = {} - kwargs = { - "runner": make_fake(Driver, seen), - "use_proxy": True, - "record_result_payloads": True, - "python_bin": sys.executable, - "server_command": ["/ext/bin/foreign-mcp", "stdio", "--mode", "candidate"], - } - if Driver is CodexCliDriver: - kwargs["allow_live"] = True - kwargs[bin_key] = "fake-bin" - driver = Driver(**kwargs) - driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - blob = json.dumps(seen) - assert "foreign-mcp" in blob or "foreign-mcp" in seen.get("cmd_joined", "") - assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") - - -def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") - - for Driver in (AntigravityCliDriver, OpencodeCliDriver): - d = Driver(runner=fake_run, use_proxy=False, python_bin=sys.executable) - run = d.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - assert run.call_source != "proxy" - - -def test_model_tiers_resolve_per_driver_and_provider(): - assert resolve_model_for_driver("api", "standard", provider="anthropic") == "claude-sonnet-5" - assert resolve_model_for_driver("api", "fast", provider="anthropic") == "claude-haiku-4-5" - assert resolve_model_for_driver("api", "standard", provider="openai") == "gpt-5.6-sol" - assert resolve_model_for_driver("api", "fast", provider="openai") == "gpt-5.6-luna" - assert resolve_model_for_driver("claude-cli", "standard") == "sonnet" - assert resolve_model_for_driver("claude-cli", "fast") == "haiku" - assert resolve_model_for_driver("codex-cli", "standard") == "gpt-5.6-sol" - assert resolve_model_for_driver("codex-cli", "fast") == "gpt-5.6-luna" - assert resolve_model_for_driver("antigravity-cli", "standard") == "gemini-3.6-flash-high" - assert resolve_model_for_driver("antigravity-cli", "fast") == "gemini-3.6-flash-low" - - -@pytest.mark.parametrize( - ("driver", "model"), - [ - ("api", "claude-opus-5"), - ("api", "sonnet"), - ("claude-cli", "sonnet"), - ("codex-cli", "sonnet"), - ("antigravity-cli", "gemini-3.1-pro-high"), - ("opencode-cli", "haiku"), - ], -) -def test_non_tier_model_strings_pass_through_unchanged(driver, model): - assert resolve_model_for_driver(driver, model) == model - - -def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): - with pytest.raises(ValueError, match=r"opencode models"): - resolve_model_for_driver("opencode-cli", "standard") - - -def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path: Path, capsys): - out = tmp_path / "must-not-exist.jsonl" - - rc = eval_main( - [ - "--driver", - "opencode-cli", - "--model", - "standard", - "--tasks", - "R1", - "--out", - str(out), - ] - ) - - assert rc == 2 - assert "explicit provider/model ID" in capsys.readouterr().err - assert out.exists() is False - - -def test_tier_mapping_is_scoped_to_cli_provider(): - with pytest.raises(ValueError, match=r"codex-cli.*anthropic.*explicit model ID"): - resolve_model_for_driver("codex-cli", "standard", provider="anthropic") - - -def test_qualified_model_id_passes_through_unchanged(): - assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" - - -def test_ensure_proxy_pythonpath_injects_repo(): - env = ensure_proxy_pythonpath({}) - assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) - # Idempotent - env2 = ensure_proxy_pythonpath(env) - assert env2["PYTHONPATH"].count(str(REPO)) == 1 - - -def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): - real_home = tmp_path / "real" - cli = real_home / ".gemini" / "antigravity-cli" - cli.mkdir(parents=True) - token_path = cli / "antigravity-oauth-token" - token_path.write_text("secret", encoding="utf-8") - # Snapshot real home before setup — must be byte-identical after. - before = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} - - fake = tmp_path / "fake" - prepare_antigravity_fake_home( - fake, - command="python", - args=["-m", "evals.proxy", "--log", "s", "--", "x"], - env={"PLANE_API_KEY": "k"}, - real_home=real_home, - ) - p1 = fake / ".gemini" / "config" / "mcp_config.json" - p2 = fake / ".gemini" / "antigravity-cli" / "mcp_config.json" - assert p1.is_file() and p2.is_file() - fake_cli = fake / ".gemini" / "antigravity-cli" - assert fake_cli.is_dir() and not fake_cli.is_symlink() - # Auth artifact is a plain COPY — never a symlink (no write-through path). - token = fake_cli / "antigravity-oauth-token" - assert token.is_file() and not token.is_symlink() - assert token.read_text(encoding="utf-8") == "secret" - # Writing the fake token must not mutate the real one. - token.write_text("mutated", encoding="utf-8") - assert token_path.read_text(encoding="utf-8") == "secret" - # mcp_config is a real file in the fake tree, not inside real home. - assert not (cli / "mcp_config.json").exists() - data = json.loads(p1.read_text()) - assert data["mcpServers"]["plane"]["command"] == "python" - # Real home byte-for-byte untouched (including oauth token). - after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} - assert after == before - - def test_process_buffer_partial_line_and_multi_line_chunk(tmp_path: Path): """Partial line stays buffered; two lines in one chunk both process.""" import os @@ -1398,139 +709,6 @@ def test_proxy_child_env_pythonpath_clean(tmp_path: Path): assert b"bad=False" in proc.stdout -def test_timeout_harvests_sidecar_calls(tmp_path: Path): - """Claude timeout path must include sidecar calls made before the timeout.""" - side_calls = [ - { - "tool": "pre_timeout", - "args": {"a": 1}, - "is_error": False, - "result_chars": 3, - "duration_ms": 1, - "seq": 1, - }, - { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - }, - ] - - def fake_run(cmd, **kwargs): - # Plant a complete sidecar next to the mcp config (temp dir still alive). - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - # Sidecar path is in the same temp dir as mcp.json for Claude. - # Find sidecar from proxy args in mcp config. - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - log_idx = args.index("--log") + 1 - side = Path(args[log_idx]) - side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "pre_timeout" - - -def test_timeout_harvest_waits_for_delayed_meta(tmp_path: Path): - """After CLI kill, harvest must poll until proxy_meta appears (not read early).""" - import threading - import time as time_mod - - call_row = { - "tool": "late_meta_tool", - "args": {"n": 1}, - "is_error": False, - "result_chars": 2, - "duration_ms": 1, - "seq": 1, - } - meta_row = { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - "pumps_alive": False, - } - seen: dict = {"waited": False} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - # Call row first — no meta yet (simulates proxy still finalizing). - side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") - - def write_meta_later() -> None: - time_mod.sleep(0.45) - with side.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(meta_row) + "\n") - seen["waited"] = True - - threading.Thread(target=write_meta_later, daemon=True).start() - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - t0 = time_mod.monotonic() - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - elapsed = time_mod.monotonic() - t0 - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "late_meta_tool" - assert seen["waited"] is True - # Must have waited for the delayed meta (~0.45s), not returned instantly. - assert elapsed >= 0.4 - assert "proxy_meta_wait_timeout" not in run.notes - - -def test_wait_for_proxy_meta_unit(tmp_path: Path): - side = tmp_path / "s.jsonl" - side.write_text("", encoding="utf-8") - assert wait_for_proxy_meta(side, max_wait_s=0.15, poll_s=0.05) is False - side.write_text(json.dumps({"row_type": "proxy_meta", "pending_left": 0}) + "\n", encoding="utf-8") - assert wait_for_proxy_meta(side, max_wait_s=1.0, poll_s=0.05) is True - - -def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): - """If meta never arrives, harvest still returns with incomplete note.""" - side = tmp_path / "s.jsonl" - side.write_text( - json.dumps({"tool": "only", "args": {}, "seq": 1, "is_error": False, "result_chars": 0}) + "\n", - encoding="utf-8", - ) - notes: list[str] = [] - calls, _client, src = harvest_proxy_after_cli_timeout([], [], side, notes, max_wait_s=0.25) - assert "proxy_meta_wait_timeout" in notes - assert len(calls) == 1 - assert src == "proxy" - assert any("incomplete" in n for n in notes) - - def test_rapid_response_pairing(tmp_path: Path): """Record-before-forward: fast child responses must pair with requests (no unmatched). @@ -1738,153 +916,6 @@ def test_bounded_shutdown_wall_clock(tmp_path: Path): assert rows[-1].get("row_type") == "proxy_meta" -def test_antigravity_fallback_runner_timeout_harvests(tmp_path: Path): - """TypeError fallback path's TimeoutExpired must still harvest via wait-for-meta.""" - call_row = { - "tool": "g_tool", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - meta = { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - } - - def fake_run(cmd, **kwargs): - run_env = kwargs.get("env") or {} - home = run_env.get("HOME") - if home: - # First attempt includes env= — plant sidecar from dual-written mcp config, - # then reject env so the driver retries without it. - for rel in ( - Path(home) / ".gemini" / "config" / "mcp_config.json", - Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", - ): - if rel.is_file(): - cfg = json.loads(rel.read_text()) - args = cfg["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - side.write_text( - "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", - encoding="utf-8", - ) - break - raise TypeError("runner does not accept env=") - # Fallback call (no env) times out — outer except must still harvest. - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "g_tool" - - -def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - - ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert str(REPO) in seen["env"].get("PYTHONPATH", "") - - -def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): - """--server-cmd must not be Claude-only.""" - from evals.runner import live as run_mod - - captured: dict = {} - - def fake_get_driver(name, **kwargs): - captured["name"] = name - captured["kwargs"] = kwargs - - class Dummy: - def run_task(self, *a, **k): - return AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - call_source="json", - ) - - return Dummy() - - monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) - - import asyncio - - async def _verify(*a, **k): - return False, "n" - - task = { - "id": "T", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": _verify, - } - rc = asyncio.run( - run_mod.run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=tmp_path / "o.jsonl", - driver_name="opencode-cli", - server_cmd=["/bin/foreign", "stdio"], - ) - ) - assert rc == 0 - assert captured["name"] == "opencode-cli" - assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] - - def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path: Path): """Proxy os.setsid() detaches from the CLI process group. diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py new file mode 100644 index 00000000..cc522253 --- /dev/null +++ b/tests/evals/test_results.py @@ -0,0 +1,359 @@ +"""Offline eval tests for results.""" + +from __future__ import annotations + +from collections import deque +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import Any + +from evals.drivers import ( + ApiDriver, +) +from evals.drivers.api import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, +) +from evals.results import RESULT_SCHEMA_VERSION, AgentRun, CallRecord, TaskResult, Usage, agent_run_to_harness_dict +from evals.runner.live import classify_call +from evals.token_counting import estimate_result_tokens +from evals.tool_names import ( + normalize_tool_call, +) + + +class FakeBackend: + provider = "fake" + model = "fake-requested" + actual_model = "fake-actual" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = deque(turns) + self.started: tuple[str | None, str, list[ToolSpec]] | None = None + self.added_results: list[list[ToolResult]] = [] + self.num_turns = 0 + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.started = (system, prompt, tools) + + def next_turn(self) -> Turn: + self.num_turns += 1 + if not self.turns: + raise AssertionError("driver requested an unexpected backend turn") + return self.turns.popleft() + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.added_results.append(results) + + +class FakeMcpSession: + def __init__(self, results: list[Any] | None = None) -> None: + self.results = deque(results or []) + self.initialized = False + self.called: list[tuple[str, dict[str, Any]]] = [] + + async def initialize(self) -> None: + self.initialized = True + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace( + name="lookup", + description="Look something up", + inputSchema={"type": "object", "properties": {"q": {"type": "string"}}}, + ), + SimpleNamespace( + name="write", + description="Write something", + inputSchema={"type": "object"}, + ), + ] + ) + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> Any: + self.called.append((name, arguments)) + if not self.results: + raise AssertionError(f"no fake result left for {name}") + return self.results.popleft() + + +def make_driver(backend: FakeBackend, session: FakeMcpSession) -> ApiDriver: + @asynccontextmanager + async def session_factory(_params): + yield session + + return ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + mcp_session_factory=session_factory, + ) + + +def run_driver(driver: ApiDriver, *, max_turns: int = 5): + return driver.run_task( + "do it", + {"SAFE": "1"}, + "fake-requested", + max_turns, + system="system", + ) + + +def test_api_driver_maps_every_legacy_row_field(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=Usage(4, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", + ), + ] + ) + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) + row = agent_run_to_harness_dict( + run, + optimal={"lookup"}, + alternate=set(), + classify=lambda tool, optimal, alternate: ( + "optimal" if tool in optimal else "alternate" if tool in alternate else "out_of_set" + ), + ) + + required = { + "final_text", + "calls", + "num_calls", + "errored_calls", + "alternate_calls", + "out_of_set_calls", + "total_result_tokens", + "usage_per_iteration", + "cum_input_tokens", + "wall_time_s", + "stop_reason", + "provider_stop_reason", + "hit_max_iterations", + "result_pair_mismatch", + "token_count_failures", + } + assert required <= row.keys() + assert { + "tool", + "class", + "args_chars", + "result_tokens", + "result_chars", + "result_kind", + "is_error", + } <= row["calls"][0].keys() + assert row["calls"][0]["result_chars"] == 5 + assert row["calls"][0]["result_tokens"] == estimate_result_tokens(5) == 2 + assert row["result_tokens_estimated"] is True + assert row["provider"] == "fake" + assert row["model"] == "fake-actual" + assert row["requested_model"] == "fake-requested" + assert row["provider_stop_reason"] == "fake_done" + + +def test_agent_run_dict_keeps_action_arg(): + run = AgentRun( + calls=[ + {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, + {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, + ], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + d = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=lambda t, o, a: "out_of_set", + ) + assert d["calls"][0]["action"] == "create" + assert "action" not in d["calls"][1] + + +def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): + """F1: ToolSearch must not inflate out_of_set or num_calls.""" + run = AgentRun( + calls=[ + normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), + ], + client_tool_calls=[ + normalize_tool_call("ToolSearch", {"query": "work items"}), + ], + final_text="done", + usage={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_cost_usd": 0.29, + "modelUsage": { + "claude-sonnet": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.29, + } + }, + }, + usage_total={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_input_tokens_including_cache": 10 + 250433 + 33838, + "total_cost_usd": 0.29, + "source": "modelUsage", + }, + stopped_reason="end_turn", + usage_scope="run", + call_source="transcript", + hit_max_turns=False, + wall_time_s=1.5, + ) + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate={"get_work_item"}, + classify=classify_call, + ) + assert out["num_calls"] == 1 + assert out["out_of_set_calls"] == 0 + assert out["calls"][0]["class"] == "optimal" + assert out["client_tool_call_count"] == 1 + assert out["client_tool_calls"][0]["tool"] == "ToolSearch" + # F2: cum_input_tokens null — not the misleading uncached-only 10 + assert out["cum_input_tokens"] is None + assert out["cum_input_tokens_reason"] + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["usage_per_iteration"] == [] + assert out["calls"][0]["result_tokens"] == 0 + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + assert "result_tokens_skipped_reason" not in out + + +def test_agent_run_hit_max_maps_to_hit_max_iterations(): + run = AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + hit_max_turns=True, + call_source="json", + ) + out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) + assert out["hit_max_iterations"] is True + assert out["stop_reason"] == "max_turns" + + +def test_agent_run_to_harness_dict_does_not_guess_usage_total(): + """Generic row mapping must not invent usage_total from a vendor usage dict. + + Drivers own normalization (ClaudeCliDriver via normalize_claude_usage, + CodexCliDriver builds its own). Missing usage_total stays None. + """ + run = AgentRun( + calls=[], + final_text="ok", + usage={ + "input_tokens": 5000, + "output_tokens": 200, + # Codex-ish shape — not Claude modelUsage. A Claude rebuild would + # silently produce a wrong / empty total if reintroduced. + "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, + }, + usage_total=None, + stopped_reason="completed", + usage_scope="run", + call_source="stream", + ) + out = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=classify_call, + ) + assert out["usage"] == run.usage + assert out["usage_total"] is None + + +def test_agent_run_to_harness_propagates_proxy_fields(): + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {"q": "a"}, + "origin": "plane", + "is_error": True, + "result_chars": 99, + "duration_ms": 42, + } + ], + final_text="x", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + usage_scope="run", + ) + d = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=lambda t, o, a: "optimal", + ) + assert d["calls"][0]["is_error"] is True + assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) + assert d["calls"][0]["result_tokens_estimated"] is True + assert d["result_tokens_estimated"] is True + assert d["calls"][0]["duration_ms"] == 42 + assert d["errored_calls"] == 1 + + +def test_task_result_schema_round_trip_owns_usage_shape(): + result = TaskResult( + row_type="result", + task_id="R1", + label="local", + server="local", + calls=[ + CallRecord( + tool="find_work_items", + classification="optimal", + result_tokens=3, + result_tokens_estimated=False, + result_token_count_method="backend", + ) + ], + num_calls=1, + usage_per_iteration=[Usage(10, 2, 3, 4)], + ) + + row = result.to_row() + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["row_type"] == "result" + assert row["label"] == "local" + assert row["server"] == "local" + assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] + loaded = TaskResult.from_row(row) + assert loaded.row_type == "result" + assert loaded.calls[0].tool == "find_work_items" + assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] diff --git a/tests/evals/test_token_counting.py b/tests/evals/test_token_counting.py new file mode 100644 index 00000000..a88ef560 --- /dev/null +++ b/tests/evals/test_token_counting.py @@ -0,0 +1,86 @@ +"""Offline eval tests for token counting.""" + +from __future__ import annotations + +import sys + +from evals.results import AgentRun, agent_run_to_harness_dict +from evals.runner.live import classify_call +from evals.token_counting import estimate_result_tokens + + +def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): + class FakeEncoding: + def encode(self, text): + assert text == "serialized workspace result" + return [10, 20, 30] + + class FakeTiktoken: + @staticmethod + def get_encoding(name): + assert name == "cl100k_base" + return FakeEncoding() + + monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len("serialized workspace result"), + "result_text": "serialized workspace result", + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) + + assert out["calls"][0]["result_tokens"] == 3 + assert out["calls"][0]["result_tokens_estimated"] is False + assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" + assert out["result_tokens_estimated"] is False + assert out["result_tokens_mode"] == "measured" + assert "result_text" not in out["calls"][0] + + +def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): + monkeypatch.setitem(sys.modules, "tiktoken", None) + text = "payload without a tokenizer" + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len(text), + "result_text": text, + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) + + assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True diff --git a/tests/evals/test_tool_names.py b/tests/evals/test_tool_names.py new file mode 100644 index 00000000..8fe6068b --- /dev/null +++ b/tests/evals/test_tool_names.py @@ -0,0 +1,24 @@ +"""Offline eval tests for tool names.""" + +from __future__ import annotations + +from evals.tool_names import ( + is_plane_mcp_tool, + strip_mcp_prefix, +) + + +def test_strip_mcp_prefix(): + assert strip_mcp_prefix("mcp__plane__list_work_items") == "list_work_items" + assert strip_mcp_prefix("mcp__plane-mcp-server__find_work_items") == "find_work_items" + assert strip_mcp_prefix("list_work_items") == "list_work_items" + assert strip_mcp_prefix("Bash") == "Bash" + + +def test_is_plane_mcp_tool(): + assert is_plane_mcp_tool("mcp__plane__find_work_items") + assert is_plane_mcp_tool("mcp__plane-foo__x") + assert not is_plane_mcp_tool("ToolSearch") + assert not is_plane_mcp_tool("Bash") + assert not is_plane_mcp_tool("mcp__other__tool") + assert not is_plane_mcp_tool("find_work_items") diff --git a/tests/test_evals_debias_verifiers.py b/tests/test_evals_debias_verifiers.py deleted file mode 100644 index 3182a1b9..00000000 --- a/tests/test_evals_debias_verifiers.py +++ /dev/null @@ -1,1082 +0,0 @@ -"""Adversarial offline verifier tests for WS3 de-bias tasks + sample existing. - -For each covered verifier: (a) untouched seed end-state must FAIL, and -(b) a plausibly-wrong end state (right field wrong value / right value wrong -item) must FAIL. Fake plane clients only — no network. -""" - -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any - -import pytest - -from evals.seed import R1_TITLE, W2_TITLE, W8_TITLE -from evals.tasks.cross import verify_c2 -from evals.tasks.debias import ( - I1_TITLE, - I3_TITLE, - I4_TITLE, - L1_TITLE, - L2_TITLE, - L3_TAG_VERSION, - L4_PROP_DISPLAY, - L4_PROP_VALUE, - L5_TITLE, - verify_i1, - verify_i2, - verify_i3, - verify_i4, - verify_i5, - verify_l1, - verify_l2, - verify_l3, - verify_l4, - verify_l5, -) -from evals.tasks.read import verify_r1 -from evals.tasks.write import verify_w2, verify_w4, verify_w8 - - -class _Page: - def __init__(self, results: list[Any] | None = None): - self.results = results or [] - self.next_page_results = False - self.next_cursor = None - - -def _run(text: str = "") -> dict[str, Any]: - return {"final_text": text, "calls": []} - - -def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: - return SimpleNamespace(id=id, name=name, **kw) - - -@pytest.fixture(autouse=True) -def _no_redis(monkeypatch): - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - -# --------------------------------------------------------------------------- -# Shared fakes -# --------------------------------------------------------------------------- - - -class _WIRetrievePlane: - """work_items.retrieve + list by name; optional labels expand.""" - - def __init__( - self, - *, - by_id: dict[str, Any], - by_name: dict[str, str] | None = None, - states: list[Any] | None = None, - ): - self._by_id = by_id - self._by_name = by_name or {} - self._states = states or [] - self.work_items = SimpleNamespace( - list=self._list, - retrieve=self._retrieve, - ) - self.states = SimpleNamespace(list=lambda **kw: _Page(self._states)) - - def _list(self, **kw): - # Minimal name filter support used by _find_item_by_name. - params = kw.get("params") - name = None - if params is not None: - name = getattr(params, "name", None) or (params.get("name") if isinstance(params, dict) else None) - if name and name in self._by_name: - wid = self._by_name[name] - row = self._by_id.get(wid) or _item(wid, name) - return _Page([row]) - return _Page([]) - - def _retrieve(self, **kw): - wid = str(kw["work_item_id"]) - if wid not in self._by_id: - raise LookupError(wid) - return self._by_id[wid] - - -# --------------------------------------------------------------------------- -# I1 — priority high by UUID -# --------------------------------------------------------------------------- - - -def test_i1_untouched_urgent_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} - ok, note = await verify_i1(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i1_wrong_item_high_target_still_urgent_fails(): - async def _go(): - # Right value on the wrong item; target remains urgent. - plane = _WIRetrievePlane( - by_id={ - "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), - "wi-other": SimpleNamespace(id="wi-other", priority="high"), - } - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} - ok, note = await verify_i1(plane, ctx, _run()) - assert ok is False, note - assert "urgent" in note or "high" in note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# I2 — state by identifier in final text -# --------------------------------------------------------------------------- - - -def test_i2_untouched_empty_final_text_fails(): - async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[st], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i2_wrong_state_name_in_text_fails(): - async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[ - st, - SimpleNamespace(id="st-done", name="Done", group="completed"), - ], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("Done")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i2_exact_state_contract_passes(): - async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[st], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("state: Backlog")) - assert ok is True, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# I3 — cycle membership by UUIDs -# --------------------------------------------------------------------------- - - -class _I3Plane: - def __init__(self, cycle_item_ids: list[str]): - self.cycles = SimpleNamespace(list_work_items=lambda **kw: _Page([_item(i, f"n-{i}") for i in cycle_item_ids])) - - -def test_i3_untouched_not_on_cycle_fails(): - async def _go(): - plane = _I3Plane(["other-1", "other-2"]) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I3_TITLE: "footer-1"}, - "cycle_current_id": "cyc-1", - } - ok, note = await verify_i3(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i3_wrong_item_on_cycle_target_missing_fails(): - async def _go(): - plane = _I3Plane(["wrong-item"]) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I3_TITLE: "footer-1"}, - "cycle_current_id": "cyc-1", - } - ok, note = await verify_i3(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# I4 — label attach by UUIDs -# --------------------------------------------------------------------------- - - -def test_i4_untouched_no_label_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[])}) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I4_TITLE: "wi-4"}, - "labels": {"perf": "lab-perf"}, - } - ok, note = await verify_i4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i4_wrong_label_attached_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[SimpleNamespace(id="lab-auth")])}) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I4_TITLE: "wi-4"}, - "labels": {"perf": "lab-perf"}, - } - ok, note = await verify_i4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# I5 — priority low by UUID -# --------------------------------------------------------------------------- - - -def test_i5_untouched_none_priority_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="none")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} - ok, note = await verify_i5(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i5_wrong_value_high_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="high")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} - ok, note = await verify_i5(plane, ctx, _run()) - assert ok is False, note - assert "high" in note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# L1 — 90m worklog + summary -# --------------------------------------------------------------------------- - - -class _L1Plane: - def __init__(self, durations: list[int], summary_ids: list[str] | None = None): - self.work_items = SimpleNamespace( - work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), - ) - rows = [SimpleNamespace(work_item_id=i, duration=90) for i in (summary_ids or [])] - self.projects = SimpleNamespace(get_worklog_summary=lambda **kw: rows) - - -def test_l1_untouched_no_worklog_fails(): - async def _go(): - plane = _L1Plane([]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l1_wrong_duration_120_fails(): - async def _go(): - plane = _L1Plane([120], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("Logged 120 minutes; summary ok.")) - assert ok is False, note - assert "90" in note - - return asyncio.run(_go()) - - -def test_l1_empty_summary_with_90m_log_fails(): - """Reviewer counterexample: 90m log present but final text empty → fail.""" - - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("")) - assert ok is False, note - assert "logged-minutes" in note.lower() - - return asyncio.run(_go()) - - -def test_l1_one_hundred_ninety_minutes_fails(): - """Reviewer counterexample: English 'ninety' must not satisfy numeric duration.""" - - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1( - plane, - ctx, - _run("Logged one hundred ninety minutes. Project summary looks fine."), - ) - assert ok is False, note - assert "duration" in note.lower() or "90" in note or "1.5" in note - - return asyncio.run(_go()) - - -def test_l1_prose_with_correct_facts_but_without_contract_fails(): - """Correct facts in prose do not satisfy the explicit output contract.""" - - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("Logged 1.5 hours total.")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l1_ninety_minutes_of_work_fails_by_design(): - """Calibration: prose without contract lines fails by design.""" - - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("90 minutes of work")) - assert ok is False, note - assert "logged-minutes" in note.lower() - - return asyncio.run(_go()) - - -def test_l1_exact_duration_and_summary_contract_passes(): - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1( - plane, - ctx, - _run("logged-minutes: 90\nsummary-work-item-id: wi-l1"), - ) - assert ok is True, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# L2 — activities -# --------------------------------------------------------------------------- - - -class _L2Plane: - def __init__(self, n_activities: int): - acts = [SimpleNamespace(id=f"a{i}", verb="updated") for i in range(n_activities)] - self.work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: _Page(acts))) - - -def test_l2_untouched_empty_final_text_fails(): - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l2_contract_count_three_passes(): - """Contract line 'count: 3' truth=3 passes.""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("Saw some history.\ncount: 3")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l2_contract_count_two_fails_truth_three(): - """Contract 'count: 2' truth=3 fails.""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("count: 2")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l2_negative_contract_and_bare_fail_truth_three(): - """'-3' and 'count: -3' fail truth=3 (signed equality).""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok1, _ = await verify_l2(plane, ctx, _run("-3")) - ok2, _ = await verify_l2(plane, ctx, _run("count: -3")) - assert ok1 is False - assert ok2 is False - - return asyncio.run(_go()) - - -def test_l2_prose_only_without_contract_fails_by_design(): - """By design: prose without 'count: N' (or bare int) fails — format is part of the task.""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("There are 3 activities and some comment phrases.")) - assert ok is False, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# L3 — release tag -# --------------------------------------------------------------------------- - - -class _L3Plane: - def __init__(self, versions: list[str]): - tags = [SimpleNamespace(id=f"t-{v}", version=v) for v in versions] - self.releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page(tags))) - - -def test_l3_untouched_no_tag_fails(): - async def _go(): - plane = _L3Plane([]) - ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l3_wrong_version_tag_fails(): - async def _go(): - plane = _L3Plane(["v0.0.1", "other-rc"]) - ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) - assert ok is False, note - assert L3_TAG_VERSION in note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# L4 — customer property values -# --------------------------------------------------------------------------- - - -class _L4Plane: - def __init__(self, *, props: list[Any], values: dict[str, list[str]]): - self.customers = SimpleNamespace( - properties=SimpleNamespace(list=lambda **kw: _Page(props)), - property_values=SimpleNamespace(list=lambda **kw: values), - ) - - -def test_l4_untouched_no_property_fails(): - async def _go(): - plane = _L4Plane(props=[], values={}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l4_right_property_wrong_value_fails(): - async def _go(): - prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") - plane = _L4Plane(props=[prop], values={"prop-1": ["Startup"]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - assert L4_PROP_VALUE in note or "Startup" in note or "lack" in note - - return asyncio.run(_go()) - - -def test_l4_industry_url_type_with_enterprise_fails(): - """Reviewer counterexample: name contains Industry, URL type, value Enterprise → fail.""" - - async def _go(): - prop = SimpleNamespace( - id="prop-url", - display_name="Industry", # substring / wrong exact name - name="industry", - property_type="URL", - ) - plane = _L4Plane(props=[prop], values={"prop-url": [L4_PROP_VALUE]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l4_exact_text_enterprise_passes(): - async def _go(): - prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") - plane = _L4Plane(props=[prop], values={"prop-1": [L4_PROP_VALUE]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is True, note - assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# L5 — attachment count -# --------------------------------------------------------------------------- - - -class _L5Plane: - def __init__(self, n: int): - rows = [SimpleNamespace(id=f"att-{i}") for i in range(n)] - self.work_items = SimpleNamespace(attachments=SimpleNamespace(list=lambda **kw: _Page(rows))) - - -def test_l5_untouched_empty_final_text_fails(): - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l5_bare_zero_passes(): - """Fallback: whole-answer bare '0' still passes truth=0.""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("0")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l5_multiline_ending_count_zero_passes(): - """Multi-line answer ending with 'count: 0' passes truth=0.""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5( - plane, - ctx, - _run("No files on this work item.\ncount: 0"), - ) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l5_prose_only_without_contract_fails_by_design(): - """By design: prose without contract line fails (format instruction is part of the task).""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("There are 0 attachments.")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l5_wrong_contract_count_fails(): - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("count: 10")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_reports_contract_int_unit(): - """Direct unit cases for the contract helper.""" - from evals.tasks.answers import reports_contract_int - - assert reports_contract_int("count: 3", 3) is True - assert reports_contract_int("count: 2", 3) is False - assert reports_contract_int("-3", 3) is False - assert reports_contract_int("count: -3", 3) is False - assert reports_contract_int("0", 0) is True - assert reports_contract_int("Some prose only", 0) is False - assert reports_contract_int("preamble\ncount: 0\n", 0) is True - # Last contract line wins - assert reports_contract_int("count: 9\ncount: 3", 3) is True - assert reports_contract_int("count: 9\ncount: 3", 9) is False - - -def test_exact_line_contract_helpers_unit(): - from evals.tasks.answers import contract_values, reports_contract_value, reports_contract_values - - text = "prose mentions state Done\nSTATE: In Progress\nitem: B\nitem: A" - assert contract_values(text, "state") == ["In Progress"] - assert reports_contract_value(text, "state", "In Progress") is True - assert reports_contract_value("- state: In Progress", "state", "In Progress") is False - assert reports_contract_values(text, "item", ["A", "B"]) is True - assert reports_contract_values("item: A\nitem: A", "item", ["A"]) is False - - -# --------------------------------------------------------------------------- -# Sample of 6 existing verifiers (untouched + wrong-value) -# --------------------------------------------------------------------------- - - -class _R1Plane: - def __init__(self, state_name: str): - st = SimpleNamespace(id="st-1", name=state_name, group="started") - self.work_items = SimpleNamespace( - list=lambda **kw: _Page([_item("r1", R1_TITLE, state=st)]), - retrieve=lambda **kw: SimpleNamespace(id="r1", name=R1_TITLE, state=st), - ) - self.states = SimpleNamespace( - list=lambda **kw: _Page( - [ - st, - SimpleNamespace(id="st-2", name="Done", group="completed"), - SimpleNamespace(id="st-3", name="Backlog", group="unstarted"), - ] - ) - ) - - -def test_existing_r1_untouched_empty_text_fails(): - async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_r1_wrong_state_in_text_fails(): - async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("Done")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_r1_exact_state_contract_passes(): - async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("state: In Progress")) - assert ok is True, note - - return asyncio.run(_go()) - - -class _R2Plane: - def __init__(self, count: int): - self._count = count - self.work_items = SimpleNamespace( - list=lambda **kw: _Page([_item(f"u{i}", f"U{i}", priority="urgent") for i in range(count)]), - count=lambda **kw: ( - SimpleNamespace(total_count=count) if False else None - ), # unused; verify_r2 uses list path - ) - - -def test_existing_r2_wrong_count_in_text_fails(): - async def _go(): - # verify_r2 counts open urgent via SDK; text must match that count. - from evals.tasks.read import verify_r2 as _vr2 - - class Plane: - def __init__(self): - self.work_items = SimpleNamespace( - list=lambda **kw: _Page( - [ - _item("1", "a", priority="urgent", state=SimpleNamespace(group="started")), - _item("2", "b", priority="urgent", state=SimpleNamespace(group="started")), - _item("3", "c", priority="urgent", state=SimpleNamespace(group="started")), - _item("4", "d", priority="urgent", state=SimpleNamespace(group="started")), - ] - ) - ) - self.states = SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) - ) - - # If verifier only checks text against live count, empty/wrong text fails. - ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) - assert ok is False, note - - return asyncio.run(_go()) - - -class _W2Plane: - def __init__(self, group: str, name: str): - st = SimpleNamespace(id="st", name=name, group=group) - self.work_items = SimpleNamespace( - list=lambda **kw: _Page([_item("w2", W2_TITLE, state=st)]), - retrieve=lambda **kw: SimpleNamespace(id="w2", state=st), - ) - self.states = SimpleNamespace(list=lambda **kw: _Page([st])) - - -def test_existing_w2_untouched_not_done_fails(): - async def _go(): - plane = _W2Plane("started", "In Progress") - ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_w2_wrong_cancelled_group_fails(): - async def _go(): - plane = _W2Plane("cancelled", "Cancelled") - ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -class _W4Plane: - def __init__(self, name: str): - self.labels = SimpleNamespace( - retrieve=lambda **kw: SimpleNamespace(id=kw["label_id"], name=name), - list=lambda **kw: _Page([SimpleNamespace(id="triage-id", name=name)]), - ) - - -def test_existing_w4_untouched_still_triage_fails(): - async def _go(): - plane = _W4Plane("triage") - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_w4_wrong_name_needs_review_fails(): - async def _go(): - plane = _W4Plane("needs-review") - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -class _W8Plane: - def __init__(self, durations: list[int]): - self.work_items = SimpleNamespace( - list=lambda **kw: _Page([_item("w8", W8_TITLE)]), - work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), - ) - - -def test_existing_w8_untouched_no_log_fails(): - async def _go(): - plane = _W8Plane([]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_w8_wrong_duration_fails(): - async def _go(): - plane = _W8Plane([60]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_c2_untouched_empty_text_fails(): - async def _go(): - ctx = {"release_changelog_text": "Changelog entry one: OAuth login hardening."} - ok, note = await verify_c2(object(), ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_c2_wrong_release_name_fails(): - async def _go(): - ok, note = await verify_c2( - object(), - {"release_changelog_text": "Changelog entry one: OAuth login hardening."}, - _run("Release 9.9.9 shipped nothing useful."), - ) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_c2_exact_release_and_shipped_contract_passes(): - async def _go(): - ok, note = await verify_c2( - object(), - { - "release_changelog_text": ( - "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." - ) - }, - _run("release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff"), - ) - assert ok is True, note - - return asyncio.run(_go()) - - -# --------------------------------------------------------------------------- -# Prompt binding hard-fail + dry-run markers -# --------------------------------------------------------------------------- - - -def test_prompt_bind_strict_empty_raises(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import PromptBindError, format_task_prompt - - t = TASKS_BY_ID["I1"] - with pytest.raises(PromptBindError): - format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) - - -def test_prompt_bind_strict_exception_raises(): - from evals.tasks.prompts import PromptBindError, format_task_prompt - - def boom(_ctx): - raise RuntimeError("seed broken") - - task = { - "id": "X", - "prompt": "do {work_item_id}", - "prompt_bind": boom, - } - with pytest.raises(PromptBindError, match="prompt_bind failed"): - format_task_prompt(task, {"project_name": "P"}, strict=True) - - -def test_prompt_bind_dry_run_markers(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] - text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) - assert "" in text - assert "EVAL x" in text - - -def test_prompt_bind_strict_success(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] - text = format_task_prompt( - t, - {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, - strict=True, - ) - assert "uuid-abc" in text - assert "<" not in text - - -# --------------------------------------------------------------------------- -# Teardown deletes release_tag + customer_property -# --------------------------------------------------------------------------- - - -class _TeardownPlane: - def __init__(self): - self.deleted: list[tuple[str, str]] = [] - self.releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)]), - delete=lambda **kw: self.deleted.append(("release_tag", kw["tag_id"])), - ), - delete=lambda **kw: self.deleted.append(("release", kw.get("release_id"))), - ) - self.customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page( - [ - SimpleNamespace( - id="prop-1", - display_name=L4_PROP_DISPLAY, - name="eval-industry", - ) - ] - ), - delete=lambda **kw: self.deleted.append(("customer_property", kw["property_id"])), - ), - list=lambda **kw: _Page([]), - delete=lambda **kw: None, - ) - self.projects = SimpleNamespace(delete=lambda **kw: None) - self.workspace_work_item_types = SimpleNamespace(delete=lambda **kw: None) - self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) - - -def test_teardown_deletes_release_tag_and_customer_property(): - from evals.seed import teardown - - plane = _TeardownPlane() - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "project_name": "EVAL x", - "workspace_objects": [ - {"kind": "release_tag", "id": "tag-tracked"}, - {"kind": "customer_property", "id": "prop-tracked"}, - ], - } - teardown(plane, ctx) - kinds = {k for k, _ in plane.deleted} - assert "release_tag" in kinds - assert "customer_property" in kinds - # Tracked ids deleted - assert ("release_tag", "tag-tracked") in plane.deleted - assert ("customer_property", "prop-tracked") in plane.deleted - - -def test_preclean_removes_stale_tag_and_property(): - from evals.seed import _preclean_ws3_workspace_artifacts - - deleted: list[tuple[str, str]] = [] - - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), - delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), - ) - ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), - delete=lambda **kw: deleted.append(("prop", kw["property_id"])), - ) - ) - - _preclean_ws3_workspace_artifacts(Plane(), "ws") - assert ("tag", "t-old") in deleted - assert ("prop", "p-old") in deleted - - -def test_preclean_delete_failure_raises_for_infra_seed(): - """Found artifact that cannot be deleted must raise (harness → infra_seed).""" - from evals.seed import _preclean_ws3_workspace_artifacts - - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), - delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), - ) - ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([]), - delete=lambda **kw: None, - ) - ) - - with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): - _preclean_ws3_workspace_artifacts(Plane(), "ws") - - -def test_preclean_empty_list_is_silent(): - from evals.seed import _preclean_ws3_workspace_artifacts - - class Plane: - releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) - customers = SimpleNamespace(properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) - - _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise - - -# --------------------------------------------------------------------------- -# L2 activity-worker seed gate -# --------------------------------------------------------------------------- - - -def test_l2_activity_gate_raises_when_empty(): - """Empty activities list after comments → TaskSkipped env:no-activity-worker.""" - from types import SimpleNamespace - - from evals.seed import R5_TITLE, _gate_activity_worker - from evals.tasks.skip import TaskSkipped - - class Plane: - work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) - - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - with pytest.raises(TaskSkipped, match="env:no-activity-worker"): - _gate_activity_worker(Plane(), "ws", ctx) - - -def test_l2_activity_gate_proceeds_when_nonempty(): - from types import SimpleNamespace - - from evals.seed import R5_TITLE, _gate_activity_worker - - class Plane: - work_items = SimpleNamespace( - activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) - ) - - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - _gate_activity_worker(Plane(), "ws", ctx) # no raise diff --git a/tests/test_evals_output_contracts.py b/tests/test_evals_output_contracts.py deleted file mode 100644 index 483fe106..00000000 --- a/tests/test_evals_output_contracts.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Focused offline tests for the read-task line contracts.""" - -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any - -from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES -from evals.tasks.cross import verify_c2 -from evals.tasks.read import verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 - - -class _Page: - def __init__(self, results: list[Any]): - self.results = results - self.next_page_results = False - self.next_cursor = None - - -def _run(text: str) -> dict[str, Any]: - return {"final_text": text, "calls": []} - - -def test_r2_written_number_prose_fails_and_count_contract_passes(): - async def _go(): - state = SimpleNamespace(id="started", name="Started", group="started") - items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] - plane = SimpleNamespace( - states=SimpleNamespace(list=lambda **kwargs: _Page([state])), - work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), - ) - ctx = {"workspace_slug": "ws", "project_id": "project"} - - prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) - contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) - - assert prose_ok is False - assert contract_ok is True, note - - return asyncio.run(_go()) - - -def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): - async def _go(): - overdue = "Session cookie not rotated after login" - ctx = { - "items": {R1_TITLE: "item-1", overdue: "item-2"}, - "r4_active_item_ids": ["item-1", "item-2"], - "r4_overdue_title": overdue, - } - text = f"cycle: {CYCLE_CURRENT}\nitem: {R1_TITLE}\nitem: {overdue}\noverdue: {overdue}" - - ok, note = await verify_r4(object(), ctx, _run(text)) - keyword_only_ok, _ = await verify_r4(object(), ctx, _run(f"cycle: {CYCLE_CURRENT}\noverdue")) - - assert ok is True, note - assert keyword_only_ok is False - - return asyncio.run(_go()) - - -def test_r5_exact_comment_lines_pass_but_free_prose_does_not(): - async def _go(): - ctx = {"r5_comment_phrases": list(R5_COMMENT_PHRASES)} - contract = "\n".join(f"comment: {phrase}" for phrase in reversed(R5_COMMENT_PHRASES)) - prose = f"The discussion covered {R5_COMMENT_PHRASES[0]} and {R5_COMMENT_PHRASES[1]}." - - contract_ok, note = await verify_r5(object(), ctx, _run(contract)) - prose_ok, _ = await verify_r5(object(), ctx, _run(prose)) - - assert contract_ok is True, note - assert prose_ok is False - - return asyncio.run(_go()) - - -def test_r6_exact_project_contract_passes_and_shorthand_fails(): - async def _go(): - expected = "EVAL deadbeef B" - ctx = {"r6_more_bugs_project": expected} - - exact_ok, note = await verify_r6(object(), ctx, _run(f"project: {expected}")) - shorthand_ok, _ = await verify_r6(object(), ctx, _run("The B project has more bugs.")) - - assert exact_ok is True, note - assert shorthand_ok is False - - return asyncio.run(_go()) - - -def test_r7_transition_contract_is_structural(): - async def _go(): - states = [ - SimpleNamespace(name="Backlog"), - SimpleNamespace(name="In Progress"), - SimpleNamespace(name="Done"), - ] - plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(states))) - ctx = {"workspace_slug": "ws", "project_id": "project"} - - exact_ok, note = await verify_r7(plane, ctx, _run("transition: Done")) - prose_ok, _ = await verify_r7(plane, ctx, _run("It can move to Done.")) - - assert exact_ok is True, note - assert prose_ok is False - - return asyncio.run(_go()) - - -def test_c2_correct_changelog_prose_without_contract_fails(): - async def _go(): - changelog = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." - prose = "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff." - - ok, _ = await verify_c2(object(), {"release_changelog_text": changelog}, _run(prose)) - - assert ok is False - - return asyncio.run(_go()) diff --git a/tests/test_evals_report_ops.py b/tests/test_evals_report_ops.py deleted file mode 100644 index 62dfffe1..00000000 --- a/tests/test_evals_report_ops.py +++ /dev/null @@ -1,780 +0,0 @@ -"""Offline tests for report stats, multi-surface table, listing tokens, cleanup, meta rows.""" - -from __future__ import annotations - -import json -import math -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from evals import cleanup as cleanup_mod -from evals import report as report_mod -from evals.listing import count_tool_tokens, tool_payload_model_facing, tool_payload_wire -from evals.report import ( - ab_compare, - build_multi_surface_table, - dedupe_rows_latest, - format_surface_cell, - is_meta_row, - load_rows, - render_multi_surface_table, - sign_test_pvalue, - summarize, - wilson_interval, -) -from evals.results import RESULT_SCHEMA_VERSION, CallRecord, TaskResult, Usage -from evals.runner import ( - is_meta_or_non_task_row, - load_resume_skip_keys, - make_run_meta_row, - maybe_write_run_meta, -) - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch): - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - - -# --------------------------------------------------------------------------- -# Sign test + Wilson -# --------------------------------------------------------------------------- - - -def test_sign_test_all_positive_hand_computed(): - """n=5 non-zero, all positive → two-sided p = 2 * (1/32) = 0.0625.""" - deltas = [1.0, 2.0, 3.0, 0.5, 4.0] - p = sign_test_pvalue(deltas) - assert p == pytest.approx(2.0 * (1.0 / 32.0)) - assert p == pytest.approx(0.0625) - - -def test_sign_test_four_of_five_hand_computed(): - """n=5, k=4 positive → right tail (C(5,4)+C(5,5))/32 = 6/32; p=2*6/32=0.375.""" - deltas = [1.0, 1.0, 1.0, 1.0, -1.0] - p = sign_test_pvalue(deltas) - right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 - assert p == pytest.approx(2.0 * right) - assert p == pytest.approx(0.375) - - -def test_sign_test_drops_zeros_and_none_when_empty(): - assert sign_test_pvalue([0.0, 0.0]) is None - assert sign_test_pvalue([]) is None - # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 - assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) - - -def test_wilson_interval_bounds(): - lo, hi = wilson_interval(5, 10) - assert lo == pytest.approx(0.2366, abs=1e-4) - assert hi == pytest.approx(0.7634, abs=1e-4) - lo0, hi0 = wilson_interval(0, 10) - assert lo0 == 0.0 - assert hi0 == pytest.approx(0.27754, abs=1e-4) - assert wilson_interval(0, 0) == (0.0, 0.0) - - -# --------------------------------------------------------------------------- -# load_rows: meta skip + dedupe -# --------------------------------------------------------------------------- - - -def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): - p = tmp_path / "r.jsonl" - lines = [ - json.dumps( - { - "row_type": "meta", - "run_id": "abc", - "label": "candidate", - "battery": "deadbeef0001", - "model": "sonnet", - "driver": "claude-cli", - "git_sha": "x", - "ts": "t", - } - ), - json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id - json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), - ] - p.write_text("\n".join(lines) + "\n", encoding="utf-8") - rows = load_rows(p) - assert len(rows) == 1 - assert rows[0].task_id == "R1" - - -def test_task_result_schema_round_trip_owns_usage_shape(): - result = TaskResult( - row_type="result", - task_id="R1", - label="local", - server="local", - calls=[ - CallRecord( - tool="find_work_items", - classification="optimal", - result_tokens=3, - result_tokens_estimated=False, - result_token_count_method="backend", - ) - ], - num_calls=1, - usage_per_iteration=[Usage(10, 2, 3, 4)], - ) - - row = result.to_row() - assert row["schema_version"] == RESULT_SCHEMA_VERSION - assert row["row_type"] == "result" - assert row["label"] == "local" - assert row["server"] == "local" - assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] - loaded = TaskResult.from_row(row) - assert loaded.row_type == "result" - assert loaded.calls[0].tool == "find_work_items" - assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] - - -def test_real_historical_rows_parse_and_report_with_backward_defaults(): - fixture = Path(__file__).parent / "fixtures" / "evals_historical_rows.jsonl" - rows = load_rows(fixture) - - assert [row.schema_version for row in rows] == [0, 0] - by_task = {row.task_id: row for row in rows} - battery4 = by_task["L3"] - assert battery4.final_text == "" - assert battery4.result_tokens_estimated is None - assert battery4.alternate_calls is None - assert battery4.calls[0].result_tokens is None - assert battery4.calls[0].action == "create" - - battery5 = by_task["R2"] - assert battery5.final_text.endswith("\n4") - assert battery5.result_tokens_estimated is True - assert [call.result_tokens for call in battery5.calls] == [315, 64] - - summary = summarize(rows) - assert summary.tasks["L3"].success == "1/1" - assert summary.tasks["L3"].med_calls == 1 - assert summary.tasks["L3"].result_tokens_mode == "unavailable" - assert summary.tasks["R2"].success == "1/1" - assert summary.tasks["R2"].med_calls == 2 - assert summary.tasks["R2"].result_tokens_mode == "estimated" - - -def test_dedupe_rows_latest_pure(): - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 1}, - {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 5}, - {"task_id": "R2", "rep": 0, "label": "local", "num_calls": 3}, - ] - out = dedupe_rows_latest(rows) - assert len(out) == 2 - by_id = {r.task_id: r for r in out} - assert by_id["R1"].num_calls == 5 - assert by_id["R2"].num_calls == 3 - - -def test_summarize_aggregate_wilson_and_call_variance(): - rows = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - ] - s = summarize(rows) - assert s.tasks["R1"].n == 3 - assert s.tasks["R1"].k == 2 - assert s.tasks["R1"].calls_min == 2.0 - assert s.tasks["R1"].calls_max == 6.0 - assert s.tasks["R1"].med_calls == 4.0 - assert s.tasks["R1"].unstable is True - assert s.tasks["R2"].unstable is False - assert s.aggregate_k == 3 - assert s.aggregate_n == 4 - assert s.multi_rep is True - assert s.unstable_task_ids == ["R1"] - assert s.unstable_tasks == 1 - assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 - - -def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): - path = tmp_path / "multi.jsonl" - outcomes = { - "R1": [True, True, True], - "R2": [True, False, True], - "R3": [False, False, False], - } - rows = [ - { - "task_id": task_id, - "rep": rep, - "label": "local", - "success": success, - "num_calls": rep + 1, - "calls": [], - } - for task_id, task_outcomes in outcomes.items() - for rep, success in enumerate(task_outcomes) - ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") - - loaded = load_rows(path) - summary = summarize(loaded) - - assert len(loaded) == 9 # distinct rep keys are not deduped away - assert summary.tasks["R1"].success == "3/3" - assert summary.tasks["R1"].unstable is False - assert summary.tasks["R2"].success == "2/3" - assert summary.tasks["R2"].wilson_lo == pytest.approx(0.2077, abs=1e-4) - assert summary.tasks["R2"].wilson_hi == pytest.approx(0.9385, abs=1e-4) - assert summary.tasks["R2"].unstable is True - assert summary.tasks["R3"].success == "0/3" - assert summary.tasks["R3"].unstable is False - assert summary.unstable_task_ids == ["R2"] - - report_mod.print_table(summary, "Summary: multi.jsonl") - output = capsys.readouterr().out - assert "unstable" in output - r2_line = next(line for line in output.splitlines() if line.startswith("R2")) - assert "2/3" in r2_line - assert "[0.21,0.94]" in r2_line - assert "YES" in r2_line - assert "measured noise floor: 1 task flipped at least once" in output - assert "minimum meaningful difference: 2 tasks" in output - - -def test_single_rep_summary_rendering_is_unchanged(capsys): - rows = [{"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 2, "calls": []}] - - report_mod.print_table(summarize(rows), "Summary: sample.jsonl") - - assert capsys.readouterr().out == ( - "Summary: sample.jsonl\n" - "aggregate success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" - "task n success wilson95 med_calls opt IQR mispick err capped h_err i_err " - "med_rtok p95_rtok med_cum_in\n" - "-------------------------------------------------------------------------------------------------------------------------------\n" - "R1 1 1/1 [0.21,1.00] 2.0 1 2.0-2.0 0.0% 0 0 0 0 " - "- - 0\n" - ) - - -def test_report_marks_entirely_estimated_result_token_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - } - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "estimated" - assert summary.tasks["R1"].result_tokens_mode == "estimated" - - report_mod.print_table(summary, "estimated") - output = capsys.readouterr().out - assert "entirely estimated" in output - assert "med_rtok~" in output - assert "~12" in output - - -def test_report_marks_mixed_measured_and_estimated_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], - "result_tokens_estimated": False, - }, - { - "task_id": "R1", - "rep": 1, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - }, - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "mixed" - assert summary.tasks["R1"].result_tokens_mode == "mixed" - - report_mod.print_table(summary, "mixed") - output = capsys.readouterr().out - assert "mixed measured and estimated" in output - assert "med_rtok*" in output - - -# --------------------------------------------------------------------------- -# A/B compare -# --------------------------------------------------------------------------- - - -def test_ab_compare_paired_deltas_and_sign_test(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, - {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 - {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired - ] - cmp = ab_compare(rows_a, rows_b) - assert cmp["n_paired"] == 2 - deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} - assert deltas["R1"] == -3.0 - assert deltas["R2"] == 1.0 - assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] - assert cmp["sign_test_p"] is not None - assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 - assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 - - -def test_ab_compare_multi_rep_uses_median_successful_call_counts(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, - ] - - cmp = ab_compare(rows_a, rows_b) - - assert cmp["multi_rep"] is True - assert cmp["unstable_a"] == 1 - assert cmp["unstable_b"] == 0 - assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] - - -# --------------------------------------------------------------------------- -# Multi-surface table -# --------------------------------------------------------------------------- - - -def _synth_row( - tid: str, - *, - rep: int = 0, - success: bool = True, - num_calls: int = 2, - alt: int | None = 0, - oos: int | None = 0, - server: str = "local", - skipped: str | None = None, - error: str | None = None, - error_class: str | None = None, - label: str = "local", -) -> dict[str, Any]: - return { - "task_id": tid, - "rep": rep, - "label": label, - "success": success, - "num_calls": num_calls, - "alternate_calls": alt, - "out_of_set_calls": oos, - "server": server, - "skipped": skipped, - "error": error, - "error_class": error_class, - "calls": [], - } - - -def test_format_surface_cell_variants(): - assert format_surface_cell(None) == "—" - assert format_surface_cell(_synth_row("R1", skipped="nope")) == "skip" - assert format_surface_cell(_synth_row("R1", error="boom")) == "ERR" - assert format_surface_cell(_synth_row("R1", error_class="infra_seed", error="x")) == "ERR" - assert format_surface_cell(_synth_row("R1", success=True, num_calls=3, alt=0, oos=0)) == "✅ 3c" - assert format_surface_cell(_synth_row("R1", success=False, num_calls=4, alt=1, oos=1)) == "❌ 4c/2mp" - # external: no mispick suffix - assert format_surface_cell(_synth_row("R1", server="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" - - -def test_multi_surface_table_snapshot_with_external(): - local = [ - _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), - _synth_row("R2", label="local", success=False, num_calls=2), - ] - candidate = [ - _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), - _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), - ] - external = [ - _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), - _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), - _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), - ] - table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) - assert table["columns"] == ["local", "candidate", "akhil"] - assert "R1" in table["task_ids"] and "R3" in table["task_ids"] - assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" - assert table["cells"]["R1"]["candidate"] == "✅ 2c" - assert table["cells"]["R1"]["akhil"] == "✅ 3c" - assert table["cells"]["R2"]["candidate"] == "skip" - assert table["cells"]["R3"]["akhil"] == "ERR" - - text = render_multi_surface_table(table, markdown=False) - assert "local" in text and "candidate" in text and "akhil" in text - assert "✅ 3c" in text - assert "skip" in text - assert "ERR" in text - assert "infra 1" in text - - md = render_multi_surface_table(table, markdown=True) - assert md.startswith("| task |") - assert "| R1 |" in md - assert "---" in md - assert "**agg**" in md - - # Footer: external mispicks n/a - assert table["footer"]["akhil"]["mispicks"] is None - assert table["footer"]["local"]["mispicks"] == 1 - assert table["footer"]["akhil"]["infra_errors"] == 1 - - -def test_multi_surface_table_aggregates_reps_and_flags_unstable(): - rows = [ - _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), - _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), - _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), - _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), - _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), - _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), - ] - - table = build_multi_surface_table([("local", rows)]) - - assert table["multi_rep"] is True - assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" - assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" - assert table["footer"]["local"]["success"] == 5 - assert table["footer"]["local"]["n"] == 6 - assert table["footer"]["local"]["unstable_tasks"] == 1 - rendered = render_multi_surface_table(table) - assert "measured noise floor: 1 task flipped at least once" in rendered - assert "minimum meaningful difference: 2 tasks" in rendered - - -def test_single_rep_multi_surface_rendering_is_unchanged(): - rows = [_synth_row("R1", label="local", success=True, num_calls=2)] - - rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) - - assert rendered == ( - "task what local \n" - "-------------------------------------------------------\n" - "R1 In project P, what is the curren… ✅ 2c\n" - "-------------------------------------------------------\n" - "local success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" - ) - - -def test_report_main_table_cli(tmp_path: Path, capsys): - f1 = tmp_path / "a.jsonl" - f2 = tmp_path / "b.jsonl" - f1.write_text( - json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", - encoding="utf-8", - ) - f2.write_text( - json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", - encoding="utf-8", - ) - rc = report_mod.main(["--table", str(f1), str(f2)]) - assert rc == 0 - out = capsys.readouterr().out - assert "local" in out and "candidate" in out - assert "R1" in out - - -def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path, capsys): - f1 = tmp_path / "old.jsonl" - f2 = tmp_path / "new.jsonl" - f1.write_text( - json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", - encoding="utf-8", - ) - f2.write_text( - json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", - encoding="utf-8", - ) - - rc = report_mod.main(["--table", str(f1), str(f2)]) - - assert rc == 0 - captured = capsys.readouterr() - assert "spans battery fingerprints" in captured.err - assert "different task prompts/questions" in captured.err - - -def test_report_main_markdown_flag(tmp_path: Path, capsys): - f1 = tmp_path / "a.jsonl" - f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") - rc = report_mod.main(["--table", "--markdown", str(f1)]) - assert rc == 0 - out = capsys.readouterr().out - assert out.startswith("| task |") - assert "| R1 |" in out - assert "---" in out - - -def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): - p = tmp_path / "d.jsonl" - rows = [ - _synth_row("R1", label="local", num_calls=1, success=True), - {**_synth_row("R1", label="local", num_calls=9, success=False)}, - ] - # Both rows have the same (task_id, rep, label), so latest-wins keeps one. - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - rc = report_mod.main(["--no-dedupe", str(p)]) - assert rc == 0 - # With no-dedupe, both rows enter summarize → n=2 for R1. - # (dedupe default would leave n=1.) - out = capsys.readouterr().out - assert "R1" in out - assert "2/2" in out or "1/2" in out # one success of two - - -# --------------------------------------------------------------------------- -# Meta line -# --------------------------------------------------------------------------- - - -def test_make_run_meta_row_and_write_once(tmp_path: Path): - path = tmp_path / "out.jsonl" - meta = make_run_meta_row( - run_id="rid", - label="candidate", - server="local", - battery="abcd1234ef00", - model="sonnet", - driver="claude-cli", - git_sha="deadbeef", - ts="2026-01-01T00:00:00+00:00", - ) - assert meta["row_type"] == "meta" - assert is_meta_row(meta) - assert is_meta_or_non_task_row(meta) - assert maybe_write_run_meta(path, meta) is True - # Append a data row — a truncating rewrite on the second call would destroy it. - with path.open("a", encoding="utf-8") as fh: - fh.write(json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True}) + "\n") - assert maybe_write_run_meta(path, meta) is False # file non-empty - lines = path.read_text(encoding="utf-8").splitlines() - assert len(lines) == 2 - assert json.loads(lines[0])["row_type"] == "meta" - assert json.loads(lines[1])["task_id"] == "R1" - - -def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text( - "\n".join( - [ - json.dumps( - { - "row_type": "meta", - "label": "candidate", - "battery": "bbbbbbbbbbbb", - "model": "sonnet", - "driver": "claude-cli", - } - ), - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "candidate", - "error": None, - "error_class": None, - "success": True, - } - ), - ] - ) - + "\n", - encoding="utf-8", - ) - skip, n_skip, n_retry = load_resume_skip_keys( - p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" - ) - assert skip == {("R1", 0, "candidate")} - assert n_skip == 1 and n_retry == 0 - - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") - - -# --------------------------------------------------------------------------- -# Listing token counts (fake tools, no network) -# --------------------------------------------------------------------------- - - -def test_count_tool_tokens_fake_list(): - class T: - def __init__(self, name, desc, inp, out=None): - self.name = name - self.description = desc - self.inputSchema = inp - self.outputSchema = out - - tools = [ - T("alpha", "short", {"type": "object"}), - T( - "beta", - "longer description here", - {"type": "object", "properties": {"x": {"type": "string"}}}, - out={"type": "object"}, - ), - ] - # Fake encode: 1 token per character (deterministic, no tiktoken needed). - encode = lambda s: list(s) # noqa: E731 - rows, total_wire, total_model = count_tool_tokens(tools, encode=encode) - assert len(rows) == 2 - assert total_wire == sum(r.wire_tokens for r in rows) - assert total_model == sum(r.model_facing_tokens for r in rows) - # Tool with outputSchema has wire > model-facing. - beta = next(r for r in rows if r.name == "beta") - assert beta.has_output_schema is True - assert beta.wire_tokens > beta.model_facing_tokens - alpha = next(r for r in rows if r.name == "alpha") - assert alpha.has_output_schema is False - assert alpha.wire_tokens == alpha.model_facing_tokens - # Sorted by wire desc - assert rows[0].wire_tokens >= rows[1].wire_tokens - - wire = tool_payload_wire(tools[1]) - assert "output_schema" in wire - model = tool_payload_model_facing(tools[1]) - assert "output_schema" not in model - - -# --------------------------------------------------------------------------- -# Cleanup dry-run never deletes -# --------------------------------------------------------------------------- - - -def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): - projects = [ - SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), - SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), - SimpleNamespace(id="p3", name="Production", identifier="PROD"), - ] - delete_calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") - - def delete(self, **kwargs): - delete_calls.append(kwargs) - - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) - - rc = cleanup_mod.main([]) # dry-run - assert rc == 0 - assert delete_calls == [] - out = capsys.readouterr().out - assert "EVAL deadbeef" in out - assert "dry-run" in out - assert "Production" not in out # prefix filter - - -def test_cleanup_yes_deletes(monkeypatch, capsys): - projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] - delete_calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") - - def delete(self, **kwargs): - delete_calls.append(kwargs) - - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) - rc = cleanup_mod.main(["--yes"]) - assert rc == 0 - assert len(delete_calls) == 1 - assert delete_calls[0]["project_id"] == "p1" - - -def test_list_projects_with_prefix_filters(): - projects = [ - SimpleNamespace(id="1", name="EVAL a"), - SimpleNamespace(id="2", name="Other"), - SimpleNamespace(id="3", name="EVAL b"), - SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " - ] - calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - calls.append({"workspace_slug": workspace_slug, "params": params}) - assert params is not None - assert params.per_page == 100 - # SDK always populates next_cursor even on last page. - return SimpleNamespace( - results=projects, - next_page_results=False, - next_cursor="100:0:0", - ) - - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "3"] - assert len(calls) == 1 # one page only — no infinite loop on next_cursor - assert calls[0]["params"].cursor is None - - -def test_list_projects_two_page_pagination(): - page1 = [SimpleNamespace(id="1", name="EVAL one")] - page2 = [SimpleNamespace(id="2", name="EVAL two")] - seen_cursors: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - seen_cursors.append(getattr(params, "cursor", None)) - if params.cursor is None: - return SimpleNamespace( - results=page1, - next_page_results=True, - next_cursor="100:0:0", - ) - assert params.cursor == "100:0:0" - return SimpleNamespace( - results=page2, - next_page_results=False, - next_cursor="200:0:0", - ) - - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "2"] - assert seen_cursors == [None, "100:0:0"] From b082b06b8c9ea1aeba28af54380c211f760d5bf9 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 13 Aug 2026 20:00:13 +0530 Subject: [PATCH 24/93] Report progress while a battery runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full battery is tens of minutes long and printed nothing until each task had already finished — and stdout is block-buffered when redirected to a file, so a run in the background showed an empty log the whole way through. A stalled run and a working one looked identical, and the only way to tell was to parse the partial JSONL. Each repetition now announces itself before it starts, with its position in the run and elapsed time, and reports a running pass/fail/skip tally after. The run ends with one summary line. Every progress print flushes. --- evals/runner/live.py | 65 +++++++++++++++++++++++++++++---- tests/evals/runner/test_live.py | 53 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/evals/runner/live.py b/evals/runner/live.py index fe6771d6..ce0349c3 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -6,6 +6,7 @@ import json import os import sys +import time import uuid from datetime import datetime, timezone from pathlib import Path @@ -77,6 +78,16 @@ def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: return False +def _elapsed(since: float) -> str: + """Wall time as mm:ss (or h:mm:ss past an hour) for progress lines.""" + seconds = int(time.monotonic() - since) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours: + return f"{hours}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + def _timeout_error_message(agent: TaskResult) -> str: """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" for note in agent.driver_notes: @@ -173,7 +184,7 @@ def _seed_fixtures( except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}", flush=True) return False except Exception as exc: row.success = False @@ -183,18 +194,20 @@ def _seed_fixtures( print( f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", file=sys.stderr, + flush=True, ) if context.get("project_name"): print( f" orphaned project may remain: {context['project_name']}", file=sys.stderr, + flush=True, ) return False if "bug_type" in task_needs and not context.get("bug_type"): reason = context.get("bug_type_skip_reason") or "bug_type unavailable" row.skipped = reason row.verify_note = reason - print(f" {task['id']} rep={repetition} SKIPPED: {reason}") + print(f" {task['id']} rep={repetition} SKIPPED: {reason}", flush=True) return False return True @@ -234,6 +247,7 @@ async def _drive_agent( print( f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", file=sys.stderr, + flush=True, ) return None except Exception as exc: @@ -328,11 +342,14 @@ async def _verify_task( ) row.success = bool(ok) row.verify_note = note - print(f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}") + print( + f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}", + flush=True, + ) except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}") + print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}", flush=True) except Exception as exc: row.success = False row.error = f"{type(exc).__name__}: {exc}" @@ -499,7 +516,7 @@ async def run_live( except SystemExit as exc: print(exc, file=sys.stderr) return 2 - print(f"resume: skipping {skip_count} completed rows, retrying {retry_count}") + print(f"resume: skipping {skip_count} completed rows, retrying {retry_count}", flush=True) # First line of a new/empty file is a meta header (skipped by loaders). meta = make_run_meta_row( @@ -516,7 +533,7 @@ async def run_live( git_sha=git_revision, ) if maybe_write_run_meta(out_path, meta): - print(f"wrote meta header battery={battery} label={label}") + print(f"wrote meta header battery={battery} label={label}", flush=True) plane, workspace_slug = make_plane_client() # User chose --driver explicitly: codex live is allowed (they own the quota). @@ -538,14 +555,29 @@ async def run_live( f"requested_model={model_alias} resolved_model={model_id} " f"label={label} tasks={[task['id'] for task in tasks]} reps={reps}" ) - print(f"writing {out_path}") + print(f"writing {out_path}", flush=True) + + # A battery is tens of minutes of silence otherwise: one line before each + # repetition says what is running now, and one after says where the run is. + total_runs = len(tasks) * reps + started_at = time.monotonic() + finished = 0 + passed = 0 + skipped = 0 + failed = 0 with out_path.open("a", encoding="utf-8") as file: for task in tasks: for repetition in range(reps): if (task["id"], repetition, label) in resume_skip: - print(f" {task['id']} rep={repetition} RESUME_SKIP") + finished += 1 + print(f" {task['id']} rep={repetition} RESUME_SKIP", flush=True) continue + print( + f"[{finished + 1:>2}/{total_runs}] {task['id']} rep={repetition} " + f"running ({_elapsed(started_at)} elapsed)", + flush=True, + ) row = await _run_task_repetition( plane=plane, driver=driver, @@ -568,4 +600,21 @@ async def run_live( file.write(json.dumps(row.to_row(), default=str) + "\n") file.flush() + finished += 1 + if row.skipped: + skipped += 1 + elif row.success: + passed += 1 + else: + failed += 1 + print( + f" {finished}/{total_runs} done · {passed} pass · " + f"{failed} fail · {skipped} skip · {_elapsed(started_at)} elapsed", + flush=True, + ) + + print( + f"finished {finished}/{total_runs} in {_elapsed(started_at)}: {passed} pass, {failed} fail, {skipped} skip", + flush=True, + ) return 0 diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 9f204ba0..0862531e 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -815,3 +815,56 @@ async def _verify(*a, **k): assert rc == 0 assert captured["name"] == "opencode-cli" assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] + + +# --------------------------------------------------------------------------- +# Progress reporting +# --------------------------------------------------------------------------- + + +def test_elapsed_formats_minutes_then_hours(monkeypatch): + from evals.runner.live import _elapsed + + clock = {"now": 1000.0} + monkeypatch.setattr(runner_live.time, "monotonic", lambda: clock["now"]) + clock["now"] = 1000.0 + 9 + assert _elapsed(1000.0) == "00:09" + clock["now"] = 1000.0 + 75 + assert _elapsed(1000.0) == "01:15" + clock["now"] = 1000.0 + 3671 + assert _elapsed(1000.0) == "1:01:11" + + +def test_run_live_reports_progress_per_repetition(tmp_path: Path, monkeypatch, capsys): + """A battery is tens of minutes long; each task must announce itself when it starts. + + Without this the operator sees nothing until the whole run ends, which is how + a stalled run looks identical to a working one. + """ + out = tmp_path / "out.jsonl" + + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + return TaskResult(final_text="done", num_calls=2) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + assert rc == 0 + + printed = capsys.readouterr().out + # Position out of total, before the task runs. + assert "[ 1/2] R1 rep=0 running" in printed + assert "[ 2/2] R2 rep=0 running" in printed + # A running tally after each, and one closing summary. + assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed + assert "finished 2/2 in " in printed + assert "2 pass, 0 fail, 0 skip" in printed From 1a1a07a4dc69e572b9c00ea05bcbc9d4fda83ed4 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 13:09:43 +0530 Subject: [PATCH 25/93] Make a feature exclusion mean the feature is off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `enable_workspace_features(exclude={"customers"})` returned before calling the API, so it only ever set True. That is sound for project features only because a fresh project is created per task-rep; the workspace persists. S5 asks the agent to enable cycles, worklogs and workspace customers, and its teardown then forced customers back on — so from the second run onward one of its three clauses was already satisfied before the agent acted. Both eval workspaces read customers=True after full teardown. Omission is not exclusion for project features either: page_view defaults to True, so excluding pages by omission would have left the feature on. It worked only because the two features S5 excludes happen to default false. Excluded features are now written False. Teardown restores the value seeding found instead of forcing True, since the harness runs against an instance it does not own. The fingerprint hashes prompts and tool sets, not fixtures, so this change would otherwise redefine what S5 asks while still claiming comparability with earlier results. CATALOG_REVISION makes that visible and is bumped here. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/build.py | 6 +- evals/seed/projects.py | 77 ++++++++++++++++---------- evals/seed/remove.py | 14 +++-- evals/tasks/__init__.py | 2 + evals/tasks/catalog.py | 30 ++++++++-- tests/evals/seed/test_seed.py | 91 ++++++++++++++++++++++++++++--- tests/evals/tasks/test_catalog.py | 42 +++++++++++++- 7 files changed, 211 insertions(+), 51 deletions(-) diff --git a/evals/seed/build.py b/evals/seed/build.py index 512de3bf..e6ea30eb 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -170,7 +170,11 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) ctx["s5_left_customers_off"] = True ctx["feature_exclude"] = sorted(feature_exclude) ctx["ws_feature_exclude"] = sorted(workspace_feature_exclude) - enable_workspace_features(plane, workspace_slug, exclude=workspace_feature_exclude) + # Prior values are captured before the write so teardown restores the workspace + # rather than forcing it to whatever this run happened to need. + ctx["workspace_features_prior"] = enable_workspace_features( + plane, workspace_slug, exclude=workspace_feature_exclude + ) enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) # Labels before items so items can attach labels later if needed. diff --git a/evals/seed/projects.py b/evals/seed/projects.py index cc5a18d1..c7d6b3ff 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -88,13 +88,32 @@ def create_project_with_identifier_retry( raise last_exc +def workspace_feature_state(plane: PlaneClient, workspace_slug: str) -> dict[str, bool | None]: + """Read the workspace feature toggles this module writes, so teardown can put them back. + + The API exposes ``customers``; older payloads spell it ``is_customer_enabled``. Returns + ``None`` for a value the API did not report rather than guessing a default. + """ + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + except Exception: + return {"customers": None} + dump = features.model_dump() if hasattr(features, "model_dump") else {} + value = dump.get("customers") + if value is None: + value = dump.get("is_customer_enabled") + if value is None: + value = getattr(features, "customers", None) + return {"customers": None if value is None else bool(value)} + + def enable_workspace_features( plane: PlaneClient, workspace_slug: str, *, exclude: set[str] | frozenset[str] | None = None, -) -> None: - """Enable workspace-level feature toggles that task preconditions need. +) -> dict[str, bool | None]: + """Set workspace-level feature toggles to what task preconditions need. Gate (plane-ee): create-customer 403 when ``check_workspace_feature(slug, IS_CUSTOMER_ENABLED)`` is false — DB column @@ -105,18 +124,20 @@ def enable_workspace_features( Deliberately does **not** set ``work_item_types``: that flips workspace-vs-project type ownership and would change S1/S3 seed mode. - ``exclude`` may contain ``customers`` (S5 leaves it off for the agent to enable). + An excluded feature is written as ``False``, not left alone. A workspace outlives + every run, so omitting the write leaves whatever the last task-rep put there — + which silently satisfied S5's customers precondition on every run after the first. + + Returns the prior values so teardown can restore them; the harness runs against a + Plane instance it does not own and should not leave configuration drift behind. """ skip = set(exclude or ()) - data: dict[str, bool] = {} - if "customers" not in skip: - data["customers"] = True - if not data: - return + prior = workspace_feature_state(plane, workspace_slug) plane.workspaces.update_features( workspace_slug=workspace_slug, - data=WorkspaceFeature(**data), + data=WorkspaceFeature(customers="customers" not in skip), ) + return prior def enable_project_features( @@ -139,20 +160,21 @@ def enable_project_features( ``exclude`` is a set of feature keys to leave disabled (for S5): ``cycles``, ``modules``, ``intakes``, ``pages``, ``worklogs``. Default: enable all (other catalog tasks need them). + + An excluded feature is written as ``False`` rather than omitted. Omitting relies on + the fresh project's default being off, which is not uniformly true — + ``page_view`` defaults to ``True`` — so excluding ``pages`` by omission would leave + the feature on. """ skip = set(exclude or ()) - update_values: dict[str, bool] = {} - if "cycles" not in skip: - update_values["cycle_view"] = True - if "modules" not in skip: - update_values["module_view"] = True - if "intakes" not in skip: - update_values["intake_view"] = True - if "pages" not in skip: - update_values["page_view"] = True - if "worklogs" not in skip: - update_values["is_time_tracking_enabled"] = True + update_values: dict[str, bool] = { + "cycle_view": "cycles" not in skip, + "module_view": "modules" not in skip, + "intake_view": "intakes" not in skip, + "page_view": "pages" not in skip, + "is_time_tracking_enabled": "worklogs" not in skip, + } if update_values: plane.projects.update( workspace_slug=workspace_slug, @@ -160,15 +182,12 @@ def enable_project_features( data=UpdateProject(**update_values), ) - feature_values: dict[str, bool] = {} - if "cycles" not in skip: - feature_values["cycles"] = True - if "modules" not in skip: - feature_values["modules"] = True - if "intakes" not in skip: - feature_values["intakes"] = True - if "pages" not in skip: - feature_values["pages"] = True + feature_values: dict[str, bool] = { + "cycles": "cycles" not in skip, + "modules": "modules" not in skip, + "intakes": "intakes" not in skip, + "pages": "pages" not in skip, + } if feature_values: plane.projects.update_features( workspace_slug=workspace_slug, diff --git a/evals/seed/remove.py b/evals/seed/remove.py index 02e16aed..5fc0232a 100644 --- a/evals/seed/remove.py +++ b/evals/seed/remove.py @@ -90,17 +90,19 @@ def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") project_id = ctx.get("project_id") - # S5 left customers off (or agent enabled them): re-enable for subsequent task-reps - # on the shared eval workspace. We always set customers=True — we do not restore a - # prior false (S5's job is to leave the workspace usable for C1). - if ctx.get("s5_left_customers_off"): + # Put workspace toggles back where the run found them. Seeding writes them explicitly + # (a task may need one off), and the workspace outlives the run, so leaving this run's + # requirements behind is drift on an instance the harness does not own. + prior = ctx.get("workspace_features_prior") or {} + prior_customers = prior.get("customers") + if prior_customers is not None: try: plane.workspaces.update_features( workspace_slug=workspace_slug, - data=WorkspaceFeature(customers=True), + data=WorkspaceFeature(customers=bool(prior_customers)), ) except Exception as exc: - print(f"teardown warning: re-enable workspace customers failed: {exc}") + print(f"teardown warning: restore workspace customers={prior_customers} failed: {exc}") # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). try: diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index 7180ad0b..7c3fe368 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -11,6 +11,7 @@ word_boundary, ) from evals.tasks.catalog import ( + CATALOG_REVISION, EXPECTED_TASK_IDS, TASKS, TASKS_BY_ID, @@ -72,6 +73,7 @@ "EXPECTED_TASK_IDS", "PromptBindError", "TASKS", + "CATALOG_REVISION", "TASKS_BY_ID", "TaskSkipped", "as_id", diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index f1f2c10e..354a9658 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -79,15 +79,32 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") +CATALOG_REVISION = 2 +"""Bumped when a deliberate change to a fixture or verifier redefines what a task asks. + +The hash below covers prompts and tool sets, not ``needs`` or verifier bodies, because +prompt drift is the signal it was built to catch. That leaves a hole: correcting a seeder +changes the question a task puts to the agent while the fingerprint keeps asserting the +results are comparable. Bumping this closes the hole by making the redefinition visible. + +Revision 1 covers batteries 6-8. Revision 2 is the workspace/project feature-exclusion +correction: excluding a feature now writes it ``False`` instead of omitting the write, so +S5 is graded on three conditions the agent must actually satisfy rather than two plus one +the workspace already happened to be in. +""" + + def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: """Stable short hash of the task battery used for a run. - SHA-256 (first 12 hex chars) over a canonical serialization of every task - sorted by id: id, prompt, sorted optimal/alternate tools, and optimal_calls. + SHA-256 (first 12 hex chars) over a canonical serialization of ``CATALOG_REVISION`` + and every task sorted by id: id, prompt, sorted optimal/alternate tools, and + optimal_calls. - Ceilings (intentionally *not* covered by the hash): - - Verifier functions and ``needs`` fixtures do not alter the fingerprint — - prompt/tool-set drift is the stability signal, not seed/verify logic. + Ceilings (intentionally *not* covered per-task by the hash): + - Verifier functions and ``needs`` fixtures do not alter the fingerprint on their + own — prompt/tool-set drift is the stability signal. Bump ``CATALOG_REVISION`` + when they change in a way that redefines the question. - The hash covers the *selected* task list: ``--tasks`` subsets produce different fingerprints than a full-catalog run. """ @@ -103,7 +120,8 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "optimal_calls": t.get("optimal_calls"), } ) - blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + document = {"revision": CATALOG_REVISION, "tasks": payload} + blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index efc93137..5c6ffef9 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -191,6 +191,9 @@ def update_features(self, workspace_slug, project_id, data): return data class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": True}) + def update_features(self, workspace_slug, data): calls.append(("ws_features", data.model_dump(exclude_none=True))) return data @@ -201,17 +204,53 @@ def update_features(self, workspace_slug, data): assert ctx["feature_exclude"] == ["cycles", "worklogs"] assert ctx["ws_feature_exclude"] == ["customers"] assert ctx["s5_left_customers_off"] is True - # No workspace customers enable call (excluded → _enable_workspace_features no-ops) - assert not any(c[0] == "ws_features" for c in calls) + # Excluded features are written OFF, not omitted. The workspace outlives the run, so + # omitting the write leaves the previous rep's value and S5's precondition never holds. + ws = next(c[1] for c in calls if c[0] == "ws_features") + assert ws.get("customers") is False + assert ctx["workspace_features_prior"] == {"customers": True} upd = next(c[1] for c in calls if c[0] == "update") - assert "cycle_view" not in upd - assert "is_time_tracking_enabled" not in upd + assert upd.get("cycle_view") is False + assert upd.get("is_time_tracking_enabled") is False assert upd.get("module_view") is True feat = next(c[1] for c in calls if c[0] == "features") - assert "cycles" not in feat + assert feat.get("cycles") is False assert feat.get("modules") is True +def test_excluding_pages_turns_page_view_off_despite_its_true_default(monkeypatch): + """``page_view`` defaults to True on a fresh project, so omission is not exclusion. + + The other excludable project features default false, which is why omitting the write + happened to work for S5. Relying on that is unsound for any feature added later. + """ + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects()) + seed_mod.enable_project_features(plane, "test-ws", "proj-1", exclude={"pages"}) + + upd = next(c[1] for c in calls if c[0] == "update") + assert upd.get("page_view") is False + feat = next(c[1] for c in calls if c[0] == "features") + assert feat.get("pages") is False + assert feat.get("cycles") is True + + def test_seed_cycles_create_add_then_backdate(monkeypatch): """Sprint 12: create (active end) → add_work_items → update(end_date past). @@ -341,7 +380,13 @@ def create(**kw): assert cur_updates == [] -def test_teardown_s5_reenables_workspace_customers(monkeypatch): +@pytest.mark.parametrize("prior", [True, False]) +def test_teardown_restores_the_workspace_value_it_found(monkeypatch, prior): + """Teardown puts the toggle back, rather than forcing the value this run wanted. + + The harness runs against an instance it does not own. Forcing ``customers=True`` on + the way out is configuration drift for anyone whose workspace had it off. + """ from types import SimpleNamespace from plane.models.workspaces import WorkspaceFeature @@ -359,8 +404,38 @@ def update_features(self, workspace_slug, data): return data plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) - seed_mod.teardown(plane, {"workspace_slug": "test-ws", "s5_left_customers_off": True, "project_id": None}) - assert calls and calls[0].get("customers") is True + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": prior}, + "project_id": None, + }, + ) + assert calls and calls[0].get("customers") is prior + + +def test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown(): + """An unreadable prior value must not become a guess written back to the workspace.""" + from types import SimpleNamespace + + calls: list = [] + + class _Workspaces: + def update_features(self, workspace_slug, data): + calls.append(data) + return data + + plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": None}, + "project_id": None, + }, + ) + assert calls == [] def test_seed_enables_features_on_second_project_too(monkeypatch): diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 9e541a28..67765071 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -80,7 +80,10 @@ "L5", ) -PINNED_SYNTHETIC_BATTERY = "eea5abf36382" +# "eea5abf36382" before CATALOG_REVISION entered the payload; "232036625e00" at revision 1. +# This pin moves with every deliberate revision bump, and must not move otherwise — an +# unexplained change means the serialization drifted, which is what the pin exists to catch. +PINNED_SYNTHETIC_BATTERY = "3e9194740d73" def test_catalog_includes_design_and_extras(): @@ -242,6 +245,43 @@ def test_battery_fingerprint_stable_and_sensitive(): assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY +def test_revision_bump_changes_the_fingerprint_for_an_unchanged_catalog(): + """A fixture/verifier correction is expressible even though the hash ignores them. + + The per-task payload deliberately omits ``needs`` and verifier bodies, so without + the revision a corrected seeder would keep the old fingerprint and go on asserting + that results answering a different question are comparable. + """ + from evals.tasks import catalog + + tasks = list(catalog.TASKS) + before = battery_fingerprint(tasks) + original = catalog.CATALOG_REVISION + try: + catalog.CATALOG_REVISION = original + 1 + after = battery_fingerprint(tasks) + finally: + catalog.CATALOG_REVISION = original + + assert after != before, "bumping the revision must move the fingerprint" + assert battery_fingerprint(tasks) == before, "restoring the revision must restore it" + + +def test_fingerprint_records_the_revision_transition(): + """Pin the current value, so a future change is read as intentional, not drift. + + ``d546d3181bdb`` was the fingerprint before the revision field existed (batteries 6-8). + Revision 2 is the feature-exclusion correction, which redefines what S5 asks without + touching any prompt or tool set — exactly the change the hash could not otherwise see. + Asserting the constant rather than merely 'it changed' is what makes an unexplained + future move visible. + """ + from evals.tasks.catalog import CATALOG_REVISION + + assert CATALOG_REVISION == 2 + assert battery_fingerprint() == "1e8b5c110b8b" + + def test_battery_fingerprint_catalog_is_nonempty(): from evals.tasks.catalog import TASKS From 79ac46d70b2fb72098944c5a2764777cd9ca70cb Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 13:12:03 +0530 Subject: [PATCH 26/93] Require a plan refusal to say so before skipping a task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit is_plan_gate treated any 403 as a plan gate. Plane returns 403 for an ordinary permission denial and for the plan gates on initiatives, teamspaces and some workflow routes, in the same {"detail": ...} shape — so a genuine permission failure was recorded as an environment skip, and every report excludes skips from its denominators. The harness was hiding the class of defect it exists to find. 402 stays unambiguous: check_feature_flag returns it for a plan refusal and nothing else in this API uses it. 403 and 400 now need the refusal to name a plan limit. One deliberate reclassification: the customer 403 says "not enabled for this workspace" with no plan wording, and is a toggle the harness sets itself, so it is no longer treated as a gate. Adds the characterization matrix the function never had, built from the payloads api/ actually returns. Restoring the old implementation fails three of them. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/projects.py | 26 +++++- tests/evals/seed/test_plan_gate.py | 126 +++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 tests/evals/seed/test_plan_gate.py diff --git a/evals/seed/projects.py b/evals/seed/projects.py index c7d6b3ff..a72d0598 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -23,15 +23,33 @@ ) +# Wording a refusal uses when the workspace's plan is what stands in the way. A feature +# switched off for a project says "not enabled for this project" instead, which is a +# configuration state the harness can change and so is not a gate. +PLAN_GATE_PROSE = ("upgrade your plan", "payment required", "subscription", "not available on your") + + def is_plan_gate(exc: BaseException) -> bool: - """True only for genuine plan/subscription feature gates — not generic API failures.""" + """True only for genuine plan/subscription feature gates — not generic API failures. + + 402 is unambiguous: ``check_feature_flag`` returns it for a plan refusal and nothing + else in this API uses it. + + 403 and 400 are not. Plane raises 403 for an ordinary permission denial *and* for the + plan gates on initiatives, teamspaces and some workflow routes, in the same + ``{"detail": ...}`` shape; 400 covers every serializer validation error as well as a + few plan refusals. Treating a bare 403 as a gate meant a genuine permission failure + was recorded as an environment skip — the harness quietly hiding the class of defect + it exists to find. Those two statuses now need the refusal to say so. + """ if not isinstance(exc, HttpError): return False - if exc.status_code in (402, 403): + if exc.status_code == 402: return True + if exc.status_code not in (400, 403): + return False blob = f"{exc} {exc.response!s}".lower() - keywords = ("plan", "subscription", "upgrade", "not available on your", "feature is not enabled") - return any(keyword in blob for keyword in keywords) + return any(phrase in blob for phrase in PLAN_GATE_PROSE) def is_identifier_collision(exc: BaseException) -> bool: diff --git a/tests/evals/seed/test_plan_gate.py b/tests/evals/seed/test_plan_gate.py new file mode 100644 index 00000000..fa4896a3 --- /dev/null +++ b/tests/evals/seed/test_plan_gate.py @@ -0,0 +1,126 @@ +"""Characterization of `is_plan_gate` against the refusals Plane actually returns. + +Every payload below is the real shape from `plane-ee/apps/api/plane/api/` — the v1 +external layer the SDK talks to — rather than an invented one. A plan gate makes the +harness record an environment skip; anything else must stay a real error, so a +misclassification here either hides a defect or reports one that is not there. +""" + +from __future__ import annotations + +import pytest +from plane.errors.errors import HttpError + +from evals.seed import is_plan_gate + +# --- genuine plan refusals ------------------------------------------------------------ + +PLAN_GATES = [ + pytest.param( + HttpError("Payment required", 402, {"error": "Payment required", "error_code": 1999}), + id="402-check_feature_flag-decorator", + ), + pytest.param( + HttpError("Payment required", 402, None), + id="402-with-no-body", + ), + pytest.param( + HttpError( + "Forbidden", + 403, + {"detail": "Payment required. Upgrade your plan to access Initiatives"}, + ), + id="403-initiatives-permission-class", + ), + pytest.param( + HttpError( + "Forbidden", + 403, + {"detail": "Payment required. Upgrade your plan to access Teamspaces"}, + ), + id="403-teamspaces-permission-class", + ), + pytest.param( + HttpError("Bad request", 400, {"error": "Upgrade your plan to enable formula properties"}), + id="400-with-plan-prose", + ), +] + +# --- refusals that are NOT plan limits ------------------------------------------------- + +NOT_PLAN_GATES = [ + pytest.param( + HttpError("Forbidden", 403, {"detail": "You don't have permission to create this project"}), + id="bare-403-is-rbac-not-a-plan-limit", + ), + pytest.param( + HttpError( + "Forbidden", + 403, + {"error": "Customer feature is not enabled for this workspace"}, + ), + id="403-customer-toggle-is-configuration-the-harness-controls", + ), + pytest.param( + HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}), + id="404-worklog-toggle", + ), + pytest.param( + HttpError("Bad request", 400, {"non_field_errors": ["Cycles are not enabled for this project"]}), + id="400-cycle-toggle", + ), + pytest.param( + HttpError("Bad request", 400, {"non_field_errors": ["Modules are not enabled for this project"]}), + id="400-module-toggle", + ), + pytest.param( + HttpError("Bad request", 400, {"name": ["This field is required."]}), + id="400-ordinary-validation-error", + ), + pytest.param( + HttpError("Server error", 500, {"error": "Internal server error"}), + id="500-never-a-gate", + ), + pytest.param( + HttpError("Too many requests", 429, {"error": "Rate limit exceeded"}), + id="429-never-a-gate", + ), +] + + +@pytest.mark.parametrize("exc", PLAN_GATES) +def test_plan_refusals_are_gates(exc): + assert is_plan_gate(exc) is True + + +@pytest.mark.parametrize("exc", NOT_PLAN_GATES) +def test_other_refusals_are_not_gates(exc): + assert is_plan_gate(exc) is False + + +@pytest.mark.parametrize( + "exc", + [ + pytest.param(RuntimeError("connection reset"), id="non-http-exception"), + pytest.param(TimeoutError(), id="timeout"), + pytest.param(ValueError("upgrade your plan"), id="non-http-even-with-plan-wording"), + ], +) +def test_non_http_exceptions_are_never_gates(exc): + """A transport failure must surface as infrastructure, not be excused as a plan limit.""" + assert is_plan_gate(exc) is False + + +def test_a_bare_403_would_previously_have_been_swallowed(): + """The regression this tightening exists for. + + An RBAC denial and an initiatives plan gate are both 403 with a ``detail`` string. + Classifying on status alone turned a permission bug into an environment skip, which + reads as 'nothing to see here' in every report that excludes skips from denominators. + """ + rbac = HttpError("Forbidden", 403, {"detail": "You don't have permission to view this issue"}) + gate = HttpError("Forbidden", 403, {"detail": "Payment required. Upgrade your plan to access Initiatives"}) + + assert rbac.status_code == gate.status_code + assert is_plan_gate(rbac) is False + assert is_plan_gate(gate) is True From 6e6bacf663eb9e939b529b864a3969eb2a2d3fb1 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 13:15:03 +0530 Subject: [PATCH 27/93] Skip a task whose capability the plan excludes, instead of erroring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN.md says a plan gate is not rewritten as an agent task failure, but only the work item type seeder implemented it. A gate while seeding a release or a customer raised, was classified infra_seed, and killed the task-rep — which is what made a flag server answering every flag "on" a hard prerequisite for running the battery at all. Both seeders now record the skip the way L2 records its missing activity worker: env:plan-gated:, carrying a reason and leaving the success denominator. The rest of the battery runs. plan_gate_skips only excuses plan limits. A permission or transport failure still raises, so the harness cannot report a clean battery over fixtures that were never built. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/customers.py | 26 ++++--- evals/seed/projects.py | 26 +++++++ evals/seed/releases.py | 12 ++-- tests/evals/seed/test_gate_tolerance.py | 90 +++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 tests/evals/seed/test_gate_tolerance.py diff --git a/evals/seed/customers.py b/evals/seed/customers.py index 6ae73edc..20678718 100644 --- a/evals/seed/customers.py +++ b/evals/seed/customers.py @@ -7,23 +7,27 @@ from plane import PlaneClient from plane.models.customers import CreateCustomer, CreateCustomerRequest +from .projects import plan_gate_skips + CUSTOMER_NAME = "Acme Corp" CUSTOMER_REQUEST_NAME = "SSO support" EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: - customer = plane.customers.create( - workspace_slug=workspace_slug, - data=CreateCustomer(name=CUSTOMER_NAME), - ) - context["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} - context["workspace_objects"].append({"kind": "customer", "id": customer.id}) - request = plane.customers.requests.create( - workspace_slug=workspace_slug, - customer_id=customer.id, - data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), - ) + """Seed the L4 customer fixture, skipping the task when the plan excludes customers.""" + with plan_gate_skips("customers"): + customer = plane.customers.create( + workspace_slug=workspace_slug, + data=CreateCustomer(name=CUSTOMER_NAME), + ) + context["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} + context["workspace_objects"].append({"kind": "customer", "id": customer.id}) + request = plane.customers.requests.create( + workspace_slug=workspace_slug, + customer_id=customer.id, + data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), + ) context["customer_request"] = { "id": request.id, "name": CUSTOMER_REQUEST_NAME, diff --git a/evals/seed/projects.py b/evals/seed/projects.py index a72d0598..870f97cd 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -2,7 +2,9 @@ from __future__ import annotations +import contextlib import secrets +from collections.abc import Iterator from typing import Any from plane import PlaneClient @@ -52,6 +54,30 @@ def is_plan_gate(exc: BaseException) -> bool: return any(phrase in blob for phrase in PLAN_GATE_PROSE) +@contextlib.contextmanager +def plan_gate_skips(feature: str) -> Iterator[None]: + """Turn a plan refusal raised inside the block into a task skip. + + ``DESIGN.md`` states that a plan gate is not rewritten as an agent task failure, but + an uncaught seed exception becomes ``infra_seed`` and the whole task-rep dies. A + capability the workspace's plan does not include is an environment fact, so it is + recorded the way L2 records its missing activity worker: a skip carrying a reason, + excluded from success denominators. + + ``TaskSkipped`` is imported here rather than at module scope because + ``evals.tasks.skip`` cannot be reached without initialising ``evals.tasks``, whose + task modules import this package. + """ + from evals.tasks.skip import TaskSkipped + + try: + yield + except Exception as exc: + if is_plan_gate(exc): + raise TaskSkipped(f"env:plan-gated:{feature}") from exc + raise + + def is_identifier_collision(exc: BaseException) -> bool: """True when project create failed because the identifier is already taken. diff --git a/evals/seed/releases.py b/evals/seed/releases.py index 9dd1f4f0..342bb5d7 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -7,16 +7,20 @@ from plane import PlaneClient from plane.models.releases import CreateRelease, UpdateReleaseChangelog +from .projects import plan_gate_skips + RELEASE_NAME = "1.2.0" RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." EVALUATION_RELEASE_TAG_VERSION = "eval-rc1" def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: - release = plane.releases.create( - workspace_slug=workspace_slug, - data=CreateRelease(name=RELEASE_NAME), - ) + """Seed the C2 release fixture, skipping the task when the plan excludes releases.""" + with plan_gate_skips("releases"): + release = plane.releases.create( + workspace_slug=workspace_slug, + data=CreateRelease(name=RELEASE_NAME), + ) context["release"] = {"id": release.id, "name": RELEASE_NAME} context["workspace_objects"].append({"kind": "release", "id": release.id}) # Single changelog body; DESIGN's "2 entries" are encoded as plain-text bullets. diff --git a/tests/evals/seed/test_gate_tolerance.py b/tests/evals/seed/test_gate_tolerance.py new file mode 100644 index 00000000..e2fe1250 --- /dev/null +++ b/tests/evals/seed/test_gate_tolerance.py @@ -0,0 +1,90 @@ +"""Seeding a plan-gated capability skips the task instead of failing the run. + +`DESIGN.md` states a plan gate is not rewritten as an agent task failure. Before this, +only the work item type seeder honoured it; a gate while seeding a release or customer +raised, became `infra_seed`, and killed the task-rep. That is what made a flag server +answering everything "on" a hard prerequisite. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.seed import seed_customer, seed_release +from evals.tasks.skip import TaskSkipped + +PLAN_REFUSAL = HttpError("Payment required", 402, {"error": "Payment required", "error_code": 1999}) +RBAC_REFUSAL = HttpError("Forbidden", 403, {"detail": "You don't have permission to do this"}) + + +def _raising_client(exc: Exception) -> SimpleNamespace: + def _boom(**_kwargs: Any): + raise exc + + return SimpleNamespace( + releases=SimpleNamespace(create=_boom, changelog=SimpleNamespace(update=_boom)), + customers=SimpleNamespace(create=_boom, requests=SimpleNamespace(create=_boom)), + ) + + +@pytest.mark.parametrize( + ("seeder", "feature"), + [(seed_release, "releases"), (seed_customer, "customers")], +) +def test_plan_gate_becomes_a_skip_with_a_reason(seeder, feature): + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(TaskSkipped) as caught: + seeder(_raising_client(PLAN_REFUSAL), "ws", context) + assert caught.value.reason == f"env:plan-gated:{feature}" + + +@pytest.mark.parametrize("seeder", [seed_release, seed_customer]) +def test_a_non_gate_failure_still_raises(seeder): + """Only plan limits are excused. A permission or transport failure is a real error. + + Swallowing these would let the harness report a clean battery while the fixtures it + graded against were never built. + """ + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(HttpError): + seeder(_raising_client(RBAC_REFUSAL), "ws", context) + + +@pytest.mark.parametrize("seeder", [seed_release, seed_customer]) +def test_transport_failures_are_not_excused(seeder): + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(RuntimeError): + seeder(_raising_client(RuntimeError("connection reset")), "ws", context) + + +def test_customer_gate_does_not_leave_a_half_built_fixture(): + """A gate on the follow-up request must not leave a customer recorded as seeded. + + The customer is created, then its request is refused. Recording the customer while + the task skips would leave a verifier reading a fixture that was never finished. + """ + created: list[str] = [] + + def _create_customer(**_kwargs: Any): + created.append("customer") + return SimpleNamespace(id="cust-1") + + def _refuse(**_kwargs: Any): + raise PLAN_REFUSAL + + plane = SimpleNamespace( + customers=SimpleNamespace( + create=_create_customer, + requests=SimpleNamespace(create=_refuse), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(TaskSkipped): + seed_customer(plane, "ws", context) + + assert created == ["customer"] + assert "customer_request" not in context From cd30fdaadad18978f10ec74c06887e8914495ec0 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 13:21:26 +0530 Subject: [PATCH 28/93] Add W11: log work against a project with time tracking switched off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the catalog exercised a disabled feature, so the harness never measured what an agent does when the API refuses. W11 seeds only time tracking off and asks for the same end state as W8; the worklog endpoints 404 with "Worklog is not enabled for the project" until the agent turns it on. The prompt grants permission to enable it. Without that, an agent that reports the limitation instead of altering project-wide configuration is arguably behaving better, and passing the enable would reward overreach rather than recovery. The failure notes separate the routes to failure — still refused, enabled but nothing logged, reported instead of enabling, and silent — because "no work log" alone does not say which happened. A do-nothing agent fails, as the canary requires of every verifier. Named W11 rather than G1: the id prefix selects the module a verifier must live in, an invariant the catalog tests enforce, and this is a write task. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/build.py | 4 + evals/tasks/catalog.py | 1 + evals/tasks/write.py | 86 ++++++++++++++++++ tests/evals/tasks/test_catalog.py | 5 +- tests/evals/tasks/test_gate_recovery.py | 115 ++++++++++++++++++++++++ 5 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/evals/tasks/test_gate_recovery.py diff --git a/evals/seed/build.py b/evals/seed/build.py index e6ea30eb..9562b8c0 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -168,6 +168,10 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) feature_exclude = {"cycles", "worklogs"} workspace_feature_exclude = {"customers"} ctx["s5_left_customers_off"] = True + if "leave_worklogs_off" in needs: + # W11: only time tracking is off, so the agent meets one obstacle rather than a + # project with several unrelated features disabled. + feature_exclude |= {"worklogs"} ctx["feature_exclude"] = sorted(feature_exclude) ctx["ws_feature_exclude"] = sorted(workspace_feature_exclude) # Prior values are captured before the write so teardown restores the workspace diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 354a9658..3b5a812f 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -29,6 +29,7 @@ "W8", "W9", "W10", + "W11", "S1", "S2", "S3", diff --git a/evals/tasks/write.py b/evals/tasks/write.py index 5c08b8a2..379a13d8 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -533,6 +533,89 @@ async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup } +W11_TITLE = W8_TITLE + + +def _time_tracking_enabled(plane: Any, workspace_slug: str, project_id: str) -> bool | None: + """Whether the project has time tracking on, or None when the read itself failed.""" + try: + project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) + except Exception: + return None + value = getattr(project, "is_time_tracking_enabled", None) + return None if value is None else bool(value) + + +async def verify_w11(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W11: the work log lands even though the project shipped with time tracking off. + + Same end state as W8, reached from an obstacle: the worklog endpoint 404s with + "Worklog is not enabled for the project" until the agent turns the feature on. The + prompt authorises that, so enabling and proceeding is the expected route rather than + an overreach the verifier would be rewarding. + + The failure notes separate the ways it can go wrong, because "no work log" alone does + not say whether the agent gave up, half-finished, or claimed success it never earned. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W11_TITLE) + if item is None: + return False, f"item {W11_TITLE!r} not found" + + durations: list[int] = [] + try: + logs = plane.work_items.work_logs.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + except Exception as exc: + # The list endpoint is gated by the same toggle, so this is the diagnosis itself. + if not is_not_found(exc): + raise + return False, f"work log list still refused, so time tracking was never enabled ({exc})" + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 120 in durations: + return True, "work log duration=120 present; agent cleared the disabled feature" + + enabled = _time_tracking_enabled(plane, workspace_slug, project_id) + if enabled: + return False, f"time tracking enabled but no 120-minute log; durations={durations}" + + final_text = str(run.get("final_text") or "").lower() + mentioned = any(phrase in final_text for phrase in ("time tracking", "not enabled", "disabled")) + if mentioned: + return False, "reported the feature was off rather than enabling it, which the prompt allowed" + return False, "no work log, and the answer never mentions the disabled feature" + + +W11_TASK: dict[str, Any] = { + "id": "W11", + "tags": {"write", "gate", "tier1"}, + "prompt": ( + f"In project {{project}}, log 2 hours of work on the item titled '{W11_TITLE}'. " + "Time tracking may be switched off for this project; you have permission to turn " + "it on if you need to." + ), + # enable time tracking → find the item → create the work log. + "optimal_calls": 3, + "optimal_tools": {"update_project", "list_work_items", "create_work_log"}, + "alternate_tools": { + "update_project_features", + "get_features", + "retrieve_project", + "search_work_items", + "list_work_logs", + "retrieve_work_item", + "list_projects", + }, + "needs": {"items", "leave_worklogs_off"}, + "verify": verify_w11, +} + + async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: """W9 (extra): bulk priority change — the three non-R1 urgent titles are now high.""" workspace_slug = ctx["workspace_slug"] @@ -630,10 +713,12 @@ async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu W8_TASK, W9_TASK, W10_TASK, + W11_TASK, ] __all__ = [ + "W11_TITLE", "WRITE_TASKS", "verify_w1", "verify_w2", @@ -645,4 +730,5 @@ async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu "verify_w8", "verify_w9", "verify_w10", + "verify_w11", ] diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 67765071..119a501a 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -35,7 +35,7 @@ "C2", } -EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features +EXTRA_IDS = {"W9", "W10", "R7", "S5", "W11"} # bulk, pages, transitions, features, gate recovery ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} @@ -60,6 +60,7 @@ "W8", "W9", "W10", + "W11", "S1", "S2", "S3", @@ -279,7 +280,7 @@ def test_fingerprint_records_the_revision_transition(): from evals.tasks.catalog import CATALOG_REVISION assert CATALOG_REVISION == 2 - assert battery_fingerprint() == "1e8b5c110b8b" + assert battery_fingerprint() == "4fb3a34a7231" def test_battery_fingerprint_catalog_is_nonempty(): diff --git a/tests/evals/tasks/test_gate_recovery.py b/tests/evals/tasks/test_gate_recovery.py new file mode 100644 index 00000000..028a2a59 --- /dev/null +++ b/tests/evals/tasks/test_gate_recovery.py @@ -0,0 +1,115 @@ +"""W11: the agent must clear a disabled project feature before it can do the work. + +W8 logs two hours against a project where time tracking is already on. W11 is the same +end state reached from an obstacle — the worklog endpoints refuse until the feature is +enabled, which the prompt explicitly permits. The verifier separates the ways it can go +wrong, because "no work log" alone does not say whether the agent gave up, half-finished, +or claimed a success it never earned. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.tasks.catalog import TASKS_BY_ID +from evals.tasks.write import W11_TITLE, verify_w11 + +WORKLOG_DISABLED = HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}) + + +class _Page: + """The paginated envelope the SDK returns from a list endpoint.""" + + def __init__(self, results: list[Any] | None = None, next_page_results: bool = False) -> None: + self.results = results or [] + self.next_page_results = next_page_results + self.next_cursor = None + + +def _plane(*, logs: Any, time_tracking: bool | None = True, item: bool = True) -> SimpleNamespace: + def _list_items(**_kwargs: Any): + return _Page([SimpleNamespace(id="item-1", name=W11_TITLE)] if item else []) + + def _list_logs(**_kwargs: Any): + if isinstance(logs, Exception): + raise logs + return logs + + def _retrieve_project(**_kwargs: Any): + return SimpleNamespace(id="proj-1", is_time_tracking_enabled=time_tracking) + + return SimpleNamespace( + work_items=SimpleNamespace( + list=_list_items, + work_logs=SimpleNamespace(list=_list_logs), + ), + projects=SimpleNamespace(retrieve=_retrieve_project), + ) + + +CTX = {"workspace_slug": "ws", "project_id": "proj-1"} + + +def test_passes_when_the_work_log_landed(): + plane = _plane(logs=[SimpleNamespace(duration=120)]) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "done"})) + assert ok is True + assert "120" in note + + +def test_still_refused_means_the_feature_was_never_enabled(): + plane = _plane(logs=WORKLOG_DISABLED, time_tracking=False) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "I could not log the time."})) + assert ok is False + assert "never enabled" in note + + +def test_enabled_but_nothing_logged_is_reported_as_half_finished(): + plane = _plane(logs=[], time_tracking=True) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "Enabled time tracking."})) + assert ok is False + assert "enabled but no 120-minute log" in note + + +def test_reporting_the_limitation_is_distinguished_from_giving_up_silently(): + """Both fail, but they are different behaviours and the report should say which.""" + reported = _plane(logs=[], time_tracking=False) + ok, note = asyncio.run( + verify_w11(reported, dict(CTX), {"final_text": "Time tracking is not enabled for this project."}) + ) + assert ok is False + assert "rather than enabling it" in note + + silent = _plane(logs=[], time_tracking=False) + ok, note = asyncio.run(verify_w11(silent, dict(CTX), {"final_text": "All done!"})) + assert ok is False + assert "never mentions" in note + + +def test_a_do_nothing_agent_fails(): + """The canary asserts this for every verifier; a gate task must not pass vacuously.""" + plane = _plane(logs=[], time_tracking=False) + ok, _ = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "", "calls": []})) + assert ok is False + + +def test_an_unexpected_error_is_not_swallowed_as_a_disabled_feature(): + """Only the 'worklog disabled' 404 is read as the obstacle; anything else is a bug.""" + plane = _plane(logs=HttpError("Server error", 500, {"error": "boom"})) + with pytest.raises(HttpError): + asyncio.run(verify_w11(plane, dict(CTX), {"final_text": ""})) + + +def test_task_seeds_time_tracking_off_and_authorises_turning_it_on(): + task = TASKS_BY_ID["W11"] + assert "leave_worklogs_off" in task["needs"], "the obstacle must actually be seeded" + assert "items" in task["needs"] + # Without explicit permission, an agent that declines to change project-wide config is + # arguably behaving better, and scoring the enable as success would reward overreach. + assert "permission" in task["prompt"].lower() + assert task["optimal_calls"] > TASKS_BY_ID["W8"]["optimal_calls"], "recovery costs a call" From 7064dd9639dc612dab38d5b51e9d2e0eeea26def Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 13:24:29 +0530 Subject: [PATCH 29/93] Stop documenting the flag server as a prerequisite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was listed as required to run the battery. Since the seeders record a plan gate as a skip, it is not: a workspace on any plan runs the catalog, and the gated tasks (C2, L4, R6, S1) drop out with a reason instead of erroring. Pointing at a permissive flag server now only changes whether those four are measured or skipped. Documents the plan-gate skip reason, the distinction between a plan limit and a project feature the harness sets itself, and CATALOG_REVISION — which anyone comparing two batteries needs, since a revision bump moves the fingerprint without any prompt changing. Adds tests for the claims most costly to leave stale: a prerequisite that is no longer real turns people away from running the harness at all. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 2 +- evals/README.md | 31 ++++++++++++++++++++++++------- tests/evals/test_docs.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 tests/evals/test_docs.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index e0c4360e..32cb19bd 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -207,7 +207,7 @@ evals/ lookups.py Plane reads used to establish verifier truth skip.py task skip signal read.py R1-R7 tasks and verifiers - write.py W1-W10 tasks and verifiers + write.py W1-W11 tasks and verifiers schema.py S1-S5 tasks and verifiers cross.py C1-C2 tasks and verifiers debias.py I1-I5 and L1-L5 tasks and verifiers diff --git a/evals/README.md b/evals/README.md index 6899657c..481f1b08 100644 --- a/evals/README.md +++ b/evals/README.md @@ -141,6 +141,13 @@ fingerprints is comparing different questions, even when task IDs are the same. results from a task whose output contract changed are not directly comparable with its rows in older batteries. `evals.report --table` warns when its input rows span fingerprints. +The hash covers prompts and tool sets, not fixtures or verifier bodies — prompt drift is what +it was built to catch. That leaves a hole, because correcting a seeder changes the question a +task puts to the agent without touching either. `CATALOG_REVISION` in `tasks/catalog.py` closes +it: bump it whenever a fixture or verifier change redefines what a task asks, and the +fingerprint moves with it. Revision 1 covers batteries 6-8; revision 2 is the feature-exclusion +correction, which made S5 genuinely require all three of its conditions. + ## Running surfaces in parallel Tasks that touch **workspace-scoped** fixtures (release tags, customer properties) collide @@ -166,7 +173,7 @@ Tasks live in the `evals/tasks/` package, grouped by task class and kept beside verifiers: - `read.py`: R1-R7 -- `write.py`: W1-W10 +- `write.py`: W1-W11 - `schema.py`: S1-S5 - `cross.py`: C1-C2 - `debias.py`: I1-I5 and L1-L5 @@ -231,11 +238,15 @@ must not pre-close the cycle that the task is meant to change. Any reachable Plane works, so how you get one is your own setup and is not kept in this repo. If you run plane-ee locally, two things make it usable for evals: -- Point `FEATURE_FLAG_SERVER_BASE_URL` at a flag server that answers every flag as on. - The gated tasks (releases, customers, worklogs, work item types) need it, and the - hosted flag server has them off for a local workspace. - Raise `API_KEY_RATE_LIMIT`; a full battery makes far more API calls than the default allows. +- Optionally point `FEATURE_FLAG_SERVER_BASE_URL` at a flag server that answers every + flag as on. This is **not** required. A capability the plan excludes makes its task + record `env:plan-gated:` and drop out of the denominator, the same way L2 + handles a missing activity worker; the rest of the battery runs unaffected. Pointing at + a permissive flag server simply means those tasks are measured rather than skipped. + + Which tasks that covers: C2 (releases), L4 (customers), R6 and S1 (work item types). Keep such scripts outside version control — `localdev/` is ignored for exactly this. @@ -243,6 +254,12 @@ Keep such scripts outside version control — `localdev/` is ignored for exactly - If seeded comments do not materialize as activities, the activity-feed task self-skips with `env:no-activity-worker` rather than failing the agent. +- A capability the workspace's plan excludes self-skips with `env:plan-gated:`. + Only a refusal that names a plan limit counts: 402, or 403/400 whose body says so. A + bare 403 is an ordinary permission denial and stays a real error, because classifying it + as a gate would let a permission bug leave the denominator and read as "nothing to see". +- A feature switched **off for a project** is not a plan gate — it is configuration the + harness sets itself, and W11 exists to measure what an agent does when it meets one. - **Gated endpoints returning 402 on a workspace that should work.** Feature flags are cached per workspace, and the cache does not record which flag server answered. Any process that touches the DB while pointed at a *different* flag server than the running @@ -253,8 +270,8 @@ Keep such scripts outside version control — `localdev/` is ignored for exactly `ff_ver:` (`plane.payment.flags.cache`) and the next request refetches. Observed while creating a workspace from a Django shell; a plan/licence problem looks identical from the outside, so check this first. -- A workspace licence is **not** required for local runs: the mock flag server enables - every flag regardless, and an unlicensed workspace seeds all fixtures (verified by - canary against a workspace with no licence row). +- A workspace licence is **not** required for local runs. With a permissive flag server an + unlicensed workspace seeds every fixture (verified by canary against a workspace with no + licence row); without one, the gated tasks skip and the rest still run. - **Offline tests** cover the harness itself and need no Plane instance: `env -u REDIS_HOST -u REDIS_PORT .venv/bin/python -m pytest -q --ignore=tests/test_integration.py` diff --git a/tests/evals/test_docs.py b/tests/evals/test_docs.py new file mode 100644 index 00000000..79300920 --- /dev/null +++ b/tests/evals/test_docs.py @@ -0,0 +1,37 @@ +"""The runbook must not re-acquire claims the code has stopped making. + +Documentation drifts silently. These are the two claims that cost the most when stale: a +prerequisite that is no longer required turns people away from running the harness at all, +and a fingerprint description that omits the revision hides why results stopped comparing. +""" + +from __future__ import annotations + +from evals import REPO_ROOT + +README = (REPO_ROOT / "evals" / "README.md").read_text() +DESIGN = (REPO_ROOT / "evals" / "DESIGN.md").read_text() + + +def test_the_flag_server_is_documented_as_optional(): + """A gated capability skips its task; it does not stop the battery.""" + assert "FEATURE_FLAG_SERVER_BASE_URL" in README, "the option should still be documented" + index = README.index("FEATURE_FLAG_SERVER_BASE_URL") + paragraph = README[max(0, index - 200) : index + 500] + assert "not** required" in paragraph or "Optionally" in paragraph, ( + "the flag server stopped being a prerequisite when the seeders learned to skip; " + "the runbook must not tell people otherwise" + ) + + +def test_the_plan_gate_skip_reason_is_documented(): + assert "env:plan-gated:" in README + + +def test_the_fingerprint_revision_is_documented(): + """Anyone comparing two batteries needs to know a revision bump can move the hash.""" + assert "CATALOG_REVISION" in README + + +def test_design_still_states_the_skip_contract_the_seeders_now_implement(): + assert "not rewritten as an agent task failure" in DESIGN From 8d37e00c743f90556f1a018afbd00f60bcc6de1f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 16:20:24 +0530 Subject: [PATCH 30/93] Halve the eval test suite by folding duplicated cases together 357 collected tests -> 177, with every assertion retained. Two techniques: verifier and output-contract tests that differed only in inputs became case tables, which removes the duplicated setup outright; tests exercising genuinely different behaviours were grouped into one function per behaviour cluster, with each case kept verbatim as a named local so a failure still says what broke. Merging widens the boundary that pytest fixtures unwind at, so each case gets its own MonkeyPatch context and its own tmp_path subdirectory. Without that, one case's environment leaked into the next and two tests began passing for the wrong reason. Verified by re-running the same 140 mutations against the reduced suite: 108 killed here against 113 before. All five of the difference were checked individually against the full suite and survive on the 357-test suite too, so they are a measurement artifact of the site-to-test mapping, not lost coverage. Also trims the largest comment blocks. The driver package docstring's probed CLI details moved into the claude and codex modules rather than being deleted, since "--max-turns is hidden from --help" is an hour of rediscovery. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/__init__.py | 36 +- evals/drivers/cli/claude.py | 9 +- evals/drivers/cli/codex.py | 8 +- evals/seed/build.py | 11 +- evals/seed/projects.py | 20 +- tests/evals/drivers/test_api_driver.py | 642 ++++----- tests/evals/drivers/test_cli_driver.py | 889 +++++++------ tests/evals/drivers/test_vendors.py | 1208 ++++++++--------- tests/evals/report/test_compare.py | 132 +- tests/evals/report/test_load.py | 107 +- tests/evals/report/test_summary.py | 107 +- tests/evals/report/test_table.py | 371 +++--- tests/evals/runner/test_canary.py | 166 +-- tests/evals/runner/test_live.py | 1363 ++++++++++---------- tests/evals/runner/test_resume.py | 351 ++--- tests/evals/seed/test_plan_gate.py | 104 +- tests/evals/seed/test_seed.py | 1245 +++++++++--------- tests/evals/tasks/test_catalog.py | 268 ++-- tests/evals/tasks/test_debias_verifiers.py | 551 +++----- tests/evals/tasks/test_output_contracts.py | 245 ++-- tests/evals/tasks/test_verifiers.py | 717 +++++----- tests/evals/test_cli.py | 143 +- tests/evals/test_docs.py | 37 +- tests/evals/test_proxy.py | 958 +++++++------- tests/evals/test_results.py | 313 +++-- tests/evals/test_token_counting.py | 141 +- 26 files changed, 4970 insertions(+), 5172 deletions(-) diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index d2e01e48..c21173a9 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -1,37 +1,7 @@ -"""Agent-driver abstraction for the Plane MCP eval harness. +"""Agent drivers: run one task against a tool surface and return an ``AgentRun``. -Drivers run one task against a tool surface and return a normalized -``AgentRun``. The default ``api`` driver owns a provider-neutral model/tool -loop over an in-process MCP client. CLI drivers (``claude-cli``, ``codex-cli``, -``antigravity-cli``, ``opencode-cli``) spawn locally installed agent CLIs on -the user's subscription — no Anthropic API key required for those paths. - -Real CLI surfaces (probed on this machine, 2026-08-12): - -Claude Code (``claude`` v2.1.228): - - ``-p`` / ``--print`` headless - - ``--mcp-config `` (repeatable; ``--strict-mcp-config``) - - ``--output-format json|text|stream-json`` (print mode) - - ``--max-turns `` (print mode; *hidden* from ``--help`` but present) - - ``--model `` - - ``--permission-mode`` choices: acceptEdits, auto, bypassPermissions, - manual, dontAsk, plan - - ``--dangerously-skip-permissions``, ``--allowedTools`` / ``--allowed-tools`` - - Transcript: ``~/.claude/projects//.jsonl`` - with ``assistant`` rows whose ``message.content`` holds ``tool_use`` blocks. - - MCP tools surface as ``mcp____`` — strip for classification. - -Codex (``codex exec``): - - ``codex exec --json`` JSONL events on stdout - - ``-c key=value`` / ``--config`` for config.toml overrides (incl. mcp_servers) - - ``-m`` / ``--model`` - - Session rollouts: ``~/.codex/sessions/**/rollout-*.jsonl`` with - ``response_item`` / ``function_call`` payloads (name + arguments JSON string) - - Marked **experimental**; live runs are opt-in (metered quota). - -This package splits that surface into focused modules (driver protocol, subprocess -lifecycle, recording-proxy glue, and per-vendor drivers). Driver names and their -public parse/configuration helpers are re-exported here. +The ``api`` driver owns a provider-neutral loop; CLI drivers spawn locally installed +agent CLIs on the user's own subscription. Probed CLI details live with each vendor. """ from __future__ import annotations diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index e4b33af9..1c415583 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -1,4 +1,11 @@ -"""Claude Code CLI driver and transcript/JSON parsers.""" +"""Claude Code CLI driver and transcript/JSON parsers. + +Probed (claude v2.1.228): -p headless; --mcp-config (repeatable) + --strict-mcp-config; +--output-format json|text|stream-json; --max-turns (present but hidden from --help); +--model; --permission-mode; transcript at +~/.claude/projects//.jsonl, assistant rows carrying +tool_use blocks; MCP tools appear as mcp____. +""" from __future__ import annotations diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index add8ce6c..d132f1a2 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -1,4 +1,10 @@ -"""Codex CLI driver and JSONL/rollout parsers.""" +"""Codex CLI driver and JSONL/rollout parsers. + +Probed: codex exec --json emits JSONL on stdout; -c key=value overrides config.toml +(including mcp_servers); -m selects the model; rollouts at +~/.codex/sessions/**/rollout-*.jsonl carry response_item/function_call payloads. +Experimental — live runs are opt-in because the quota is metered. +""" from __future__ import annotations diff --git a/evals/seed/build.py b/evals/seed/build.py index 9562b8c0..e59db02a 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -153,15 +153,8 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) ctx["project_id"] = project.id ctx["project_identifier"] = getattr(project, "identifier", None) - # Feature enablement (workspace first, then project). - # - # Ordering for S5 vs C1 on a shared eval workspace: - # - Each task-rep has its own seed/teardown; there is no multi-task seed batch. - # - Default tasks: enable workspace customers=True so C1 create_customer works. - # - S5 (needs leave_cycles_worklogs_off): leave project cycles+worklogs AND - # workspace customers OFF so the agent must flip all three; teardown then - # re-enables customers=True so a later C1 rep is not left 403ing. - # - We do not try to "run workspace enable after S5 check" — seed is per-task. + # Workspace first, then project. Seeding is per task-rep, so S5 turning workspace + # customers off must be undone in teardown or a later C1 rep 403s. feature_exclude: set[str] = set() workspace_feature_exclude: set[str] = set() if "leave_cycles_worklogs_off" in needs: diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 870f97cd..0c2da6d2 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -191,24 +191,10 @@ def enable_project_features( *, exclude: set[str] | frozenset[str] | None = None, ) -> None: - """Enable per-project feature gates that fresh projects ship with disabled. + """Set per-project feature gates; ``exclude`` names the ones to leave off. - Two SDK calls (harmless if already on): - - 1. ``projects.update`` / ``UpdateProject`` — view columns API gates read: - ``cycle_view``, ``module_view``, ``intake_view``, ``page_view``, - ``is_time_tracking_enabled`` (worklog 404 when false). - 2. ``projects.update_features`` / ``ProjectFeature`` — capability flags - that the server maps onto the same view columns for cycles/modules/… . - - ``exclude`` is a set of feature keys to leave disabled (for S5): - ``cycles``, ``modules``, ``intakes``, ``pages``, ``worklogs``. - Default: enable all (other catalog tasks need them). - - An excluded feature is written as ``False`` rather than omitted. Omitting relies on - the fresh project's default being off, which is not uniformly true — - ``page_view`` defaults to ``True`` — so excluding ``pages`` by omission would leave - the feature on. + Excluded features are written ``False``, not omitted: ``page_view`` defaults to True, + so omission would silently leave it on. """ skip = set(exclude or ()) diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 9594e8e7..64257077 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -181,195 +181,198 @@ async def session_factory(_params): assert run.usage_per_iteration == [Usage(7, 2, 0, 0)] -def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], - usage=Usage(10, 2, 3, 1), - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="", - tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], - usage=Usage(20, 4, 6, 0), - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="done", - tool_calls=[], - usage=Usage(30, 6, 9, 0), - stop_reason=StopReason.END_TURN, - provider_stop_reason="fake_done", - ), - ] - ) - session = FakeMcpSession( - [ - {"content": [{"type": "text", "text": "first result"}], "isError": False}, - {"content": [{"type": "text", "text": "second"}], "isError": True}, - ] - ) - - run = run_driver(make_driver(backend, session)) - - assert session.initialized is True - assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] - assert backend.started is not None - assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] - assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] - assert run.final_text == "done" - assert run.stopped_reason == "end_turn" - assert run.cum_input_tokens == 60 - assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] - assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] - assert [call["result_tokens"] for call in run.calls] == [ - estimate_result_tokens(len("first result")), - estimate_result_tokens(len("second")), - ] - assert [call["is_error"] for call in run.calls] == [False, True] - assert run.result_tokens_estimated is True - assert run.token_count_failures == 0 - assert run.provider == "fake" - assert run.model == "fake-actual" - assert run.provider_stop_reason == "fake_done" - - -def test_api_driver_refusal_records_calls_but_executes_nothing(): - backend = FakeBackend( - [ - Turn( - text="declined", - tool_calls=[ToolCall("write-1", "write", {"value": "x"})], - usage=Usage(1, 1), - stop_reason=StopReason.REFUSAL, - ) - ] - ) - session = FakeMcpSession() - - run = run_driver(make_driver(backend, session)) - - assert [call["tool"] for call in run.calls] == ["write"] - assert session.called == [] - assert backend.added_results == [] - assert run.stopped_reason == "refusal" - assert run.hit_max_turns is False - +def test_api_driver_behaviours(): + def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], + usage=Usage(10, 2, 3, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], + usage=Usage(20, 4, 6, 0), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=Usage(30, 6, 9, 0), + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", + ), + ] + ) + session = FakeMcpSession( + [ + {"content": [{"type": "text", "text": "first result"}], "isError": False}, + {"content": [{"type": "text", "text": "second"}], "isError": True}, + ] + ) -def test_api_driver_pairs_results_by_id_not_ordinal(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), - ] - ) - # The fake session deliberately returns tagged results in reverse ID order. - session = FakeMcpSession( - [ - ToolResult(call_id="b", text="BBBB"), - ToolResult(call_id="a", text="A"), + run = run_driver(make_driver(backend, session)) + + assert session.initialized is True + assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] + assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] + assert run.final_text == "done" + assert run.stopped_reason == "end_turn" + assert run.cum_input_tokens == 60 + assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] + assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] + assert [call["result_tokens"] for call in run.calls] == [ + estimate_result_tokens(len("first result")), + estimate_result_tokens(len("second")), ] - ) - - run = run_driver(make_driver(backend, session)) - - assert [call["result_chars"] for call in run.calls] == [1, 4] - assert run.result_pair_mismatch is False - + assert [call["is_error"] for call in run.calls] == [False, True] + assert run.result_tokens_estimated is True + assert run.token_count_failures == 0 + assert run.provider == "fake" + assert run.model == "fake-actual" + assert run.provider_stop_reason == "fake_done" + + def test_api_driver_refusal_records_calls_but_executes_nothing(): + backend = FakeBackend( + [ + Turn( + text="declined", + tool_calls=[ToolCall("write-1", "write", {"value": "x"})], + usage=Usage(1, 1), + stop_reason=StopReason.REFUSAL, + ) + ] + ) + session = FakeMcpSession() + + run = run_driver(make_driver(backend, session)) + + assert [call["tool"] for call in run.calls] == ["write"] + assert session.called == [] + assert backend.added_results == [] + assert run.stopped_reason == "refusal" + assert run.hit_max_turns is False + + def test_api_driver_pairs_results_by_id_not_ordinal(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + # The fake session deliberately returns tagged results in reverse ID order. + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="a", text="A"), + ] + ) -def test_api_driver_flags_result_id_mismatch(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="done", - tool_calls=[], - usage=None, - stop_reason=StopReason.END_TURN, - ), - ] - ) - session = FakeMcpSession( - [ - ToolResult(call_id="b", text="BBBB"), - ToolResult(call_id="unknown", text="lost"), - ] - ) + run = run_driver(make_driver(backend, session)) - run = run_driver(make_driver(backend, session)) + assert [call["result_chars"] for call in run.calls] == [1, 4] + assert run.result_pair_mismatch is False - assert run.result_pair_mismatch is True - assert [call["result_chars"] for call in run.calls] == [0, 4] + def test_api_driver_flags_result_id_mismatch(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="unknown", text="lost"), + ] + ) + run = run_driver(make_driver(backend, session)) -def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="must not be read", - tool_calls=[], - usage=None, - stop_reason=StopReason.END_TURN, - ), - ] - ) - session = FakeMcpSession([ToolResult(call_id="a", text="result")]) + assert run.result_pair_mismatch is True + assert [call["result_chars"] for call in run.calls] == [0, 4] - run = run_driver(make_driver(backend, session), max_turns=1) + def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="must not be read", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession([ToolResult(call_id="a", text="result")]) - assert session.called == [("lookup", {"q": "a"})] - assert len(backend.added_results) == 1 - assert backend.num_turns == 1 - assert run.hit_max_turns is True - assert run.stopped_reason == "tool_use" + run = run_driver(make_driver(backend, session), max_turns=1) + assert session.called == [("lookup", {"q": "a"})] + assert len(backend.added_results) == 1 + assert backend.num_turns == 1 + assert run.hit_max_turns is True + assert run.stopped_reason == "tool_use" -def test_api_driver_clean_end_on_last_iteration_is_not_capped(): - backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + def test_api_driver_clean_end_on_last_iteration_is_not_capped(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) - run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) + run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) - assert run.hit_max_turns is False - assert run.stopped_reason == "end_turn" + assert run.hit_max_turns is False + assert run.stopped_reason == "end_turn" + def test_api_driver_uses_optional_backend_token_counter(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + backend.count_tokens = lambda text: len(text) + 10 -def test_api_driver_uses_optional_backend_token_counter(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), - ] - ) - backend.count_tokens = lambda text: len(text) + 10 + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) - run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) + assert run.calls[0]["result_tokens"] == 13 + assert run.result_tokens_estimated is False + assert run.token_count_failures == 0 - assert run.calls[0]["result_tokens"] == 13 - assert run.result_tokens_estimated is False - assert run.token_count_failures == 0 + test_api_driver_multi_turn_tool_loop_and_usage_accumulation() + test_api_driver_refusal_records_calls_but_executes_nothing() + test_api_driver_pairs_results_by_id_not_ordinal() + test_api_driver_flags_result_id_mismatch() + test_api_driver_iteration_cap_only_flags_mid_tool_loop() + test_api_driver_clean_end_on_last_iteration_is_not_capped() + test_api_driver_uses_optional_backend_token_counter() @pytest.mark.parametrize( @@ -495,174 +498,183 @@ def test_openai_backend_normalizes_and_preserves_stop_reason(raw_reason, expecte assert turn.provider_stop_reason == raw_reason -def test_openai_backend_translates_tools_calls_and_tool_messages(): - responses = [ - { - "model": "gpt-actual", - "choices": [ - { - "finish_reason": "tool_calls", - "message": { - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, - } - ], - }, - } - ], - "usage": { - "prompt_tokens": 12, - "completion_tokens": 3, - "prompt_tokens_details": {"cached_tokens": 5}, - }, - }, - { - "model": "gpt-actual", - "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], - "usage": {"prompt_tokens": 20, "completion_tokens": 4}, - }, - ] - completions = FakeOpenAICompletions(responses) - client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) - backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) - tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) - - backend.start("system", "prompt", [tool]) - first = backend.next_turn() - backend.add_tool_results([ToolResult("call-1", "value")]) - second = backend.next_turn() - - first_request = completions.requests[0] - assert first_request["max_completion_tokens"] == 321 - assert first_request["messages"] == [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "prompt"}, - ] - assert first_request["tools"] == [ - { - "type": "function", - "function": { - "name": "lookup", - "description": "Look up", - "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, - }, - } - ] - assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] - assert first.stop_reason is StopReason.TOOL_USE - assert first.provider_stop_reason == "tool_calls" - assert first.usage == Usage(12, 3, 5, 0) - second_messages = completions.requests[1]["messages"] - assert second_messages[2] == { - "role": "assistant", - "content": None, - "tool_calls": [ +def test_openai_backend_behaviours(): + def test_openai_backend_translates_tools_calls_and_tool_messages(): + responses = [ { - "id": "call-1", - "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, - } - ], - } - assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} - assert second.text == "done" - assert second.stop_reason is StopReason.END_TURN - assert second.provider_stop_reason == "stop" - assert backend.actual_model == "gpt-actual" - - -def test_openai_backend_normalizes_refusal_for_driver_guard(): - completions = FakeOpenAICompletions( - [ - { - "model": "gpt", + "model": "gpt-actual", "choices": [ { - "finish_reason": "content_filter", + "finish_reason": "tool_calls", "message": { "content": None, - "refusal": "declined", "tool_calls": [ { - "id": "danger", + "id": "call-1", "type": "function", - "function": {"name": "write", "arguments": "{}"}, + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, } ], }, } ], - "usage": None, + "usage": { + "prompt_tokens": 12, + "completion_tokens": 3, + "prompt_tokens_details": {"cached_tokens": 5}, + }, + }, + { + "model": "gpt-actual", + "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], + "usage": {"prompt_tokens": 20, "completion_tokens": 4}, + }, + ] + completions = FakeOpenAICompletions(responses) + client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) + backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("call-1", "value")]) + second = backend.next_turn() + + first_request = completions.requests[0] + assert first_request["max_completion_tokens"] == 321 + assert first_request["messages"] == [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "prompt"}, + ] + assert first_request["tools"] == [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, } ] - ) - backend = OpenAIBackend( - "gpt", - max_tokens=10, - client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), - ) - backend.start(None, "prompt", []) - - turn = backend.next_turn() - - assert turn.stop_reason is StopReason.REFUSAL - assert turn.provider_stop_reason == "content_filter" - assert turn.text == "declined" - assert turn.tool_calls == [ToolCall("danger", "write", {})] - - -def test_tool_spec_from_mcp_reads_dict_and_object_entries(): - from evals.drivers.driver import tool_spec_from_mcp + assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] + assert first.stop_reason is StopReason.TOOL_USE + assert first.provider_stop_reason == "tool_calls" + assert first.usage == Usage(12, 3, 5, 0) + second_messages = completions.requests[1]["messages"] + assert second_messages[2] == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + } + ], + } + assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} + assert second.text == "done" + assert second.stop_reason is StopReason.END_TURN + assert second.provider_stop_reason == "stop" + assert backend.actual_model == "gpt-actual" + + def test_openai_backend_normalizes_refusal_for_driver_guard(): + completions = FakeOpenAICompletions( + [ + { + "model": "gpt", + "choices": [ + { + "finish_reason": "content_filter", + "message": { + "content": None, + "refusal": "declined", + "tool_calls": [ + { + "id": "danger", + "type": "function", + "function": {"name": "write", "arguments": "{}"}, + } + ], + }, + } + ], + "usage": None, + } + ] + ) + backend = OpenAIBackend( + "gpt", + max_tokens=10, + client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), + ) + backend.start(None, "prompt", []) - as_dict = tool_spec_from_mcp( - {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} - ) - assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") - assert as_dict.input_schema == {"type": "object", "x": 1} + turn = backend.next_turn() - as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) - assert as_object.name == "create_cycle" - # A missing or non-dict schema must still yield a usable object schema. - assert as_object.input_schema == {"type": "object"} - assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} + assert turn.stop_reason is StopReason.REFUSAL + assert turn.provider_stop_reason == "content_filter" + assert turn.text == "declined" + assert turn.tool_calls == [ToolCall("danger", "write", {})] + test_openai_backend_translates_tools_calls_and_tool_messages() + test_openai_backend_normalizes_refusal_for_driver_guard() -def test_tool_result_from_mcp_text_only_joins_blocks(): - from evals.drivers.driver import tool_result_from_mcp - result = tool_result_from_mcp( - "call_1", - {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, - ) - assert (result.call_id, result.text, result.kind, result.is_error) == ("call_1", "first\nsecond", "text", False) +def test_tool_behaviours(): + def test_tool_spec_from_mcp_reads_dict_and_object_entries(): + from evals.drivers.driver import tool_spec_from_mcp + as_dict = tool_spec_from_mcp( + {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} + ) + assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") + assert as_dict.input_schema == {"type": "object", "x": 1} + + as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) + assert as_object.name == "create_cycle" + # A missing or non-dict schema must still yield a usable object schema. + assert as_object.input_schema == {"type": "object"} + assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} + + def test_tool_result_behaviours(): + def test_tool_result_from_mcp_text_only_joins_blocks(): + from evals.drivers.driver import tool_result_from_mcp + + result = tool_result_from_mcp( + "call_1", + {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + ) + assert (result.call_id, result.text, result.kind, result.is_error) == ( + "call_1", + "first\nsecond", + "text", + False, + ) -def test_tool_result_from_mcp_serializes_non_text_blocks(): - """A tool returning an image must not be counted as if it returned nothing. + def test_tool_result_from_mcp_serializes_non_text_blocks(): + from evals.drivers.driver import tool_result_from_mcp - This path serializes the whole content list, so it is the only one that - needs ``json`` at run time — a missing import here fails no other test. - """ - from evals.drivers.driver import tool_result_from_mcp + mixed = tool_result_from_mcp( + "call_2", + {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, + ) + assert mixed.kind == "mixed" + assert '"image"' in mixed.text and "chart:" in mixed.text - mixed = tool_result_from_mcp( - "call_2", - {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, - ) - assert mixed.kind == "mixed" - assert '"image"' in mixed.text and "chart:" in mixed.text + image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) + assert image_only.kind == "image" + assert '"data":"AAAA"' in image_only.text - image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) - assert image_only.kind == "image" - assert '"data":"AAAA"' in image_only.text + def test_tool_result_from_mcp_propagates_error_flag_in_both_spellings(): + from evals.drivers.driver import tool_result_from_mcp + assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True + assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True -def test_tool_result_from_mcp_propagates_error_flag_in_both_spellings(): - from evals.drivers.driver import tool_result_from_mcp + test_tool_result_from_mcp_text_only_joins_blocks() + test_tool_result_from_mcp_serializes_non_text_blocks() + test_tool_result_from_mcp_propagates_error_flag_in_both_spellings() - assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True - assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True + test_tool_spec_from_mcp_reads_dict_and_object_entries() + test_tool_result_behaviours() diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index dc3e51d6..173ceda5 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -42,57 +42,120 @@ def _pid_alive(pid: int) -> bool: REPO = Path(__file__).resolve().parents[3] -def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path: Path): - """Timeout kills the whole process group, not just the parent (codex node→native case). +def test_run_behaviours(tmp_path, monkeypatch): + def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path): + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_cli.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + # Grandchild stays in the same process group (no start_new_session). + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + # Hold our stdout open forever (simulates grandchild pipe hold). + time.sleep(9999) + """ + ), + encoding="utf-8", + ) - Sticky CLI: parent spawns a grandchild in the same group that would keep - stdout open if only the parent were killed. Assert the runner returns - quickly and both PIDs are dead. - """ - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky_cli.py" - script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - pidfile = Path({str(pidfile)!r}) - # Grandchild stays in the same process group (no start_new_session). - child = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(9999)"], + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired) as ei: + run_cli_subprocess( + [sys.executable, str(script)], + timeout=1.0, + capture_output=True, + text=True, ) - pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") - # Hold our stdout open forever (simulates grandchild pipe hold). - time.sleep(9999) - """ - ), - encoding="utf-8", - ) + elapsed = time.monotonic() - t0 + assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" + assert getattr(ei.value, "killed_process_group", False) is True - t0 = time.monotonic() - with pytest.raises(subprocess.TimeoutExpired) as ei: - run_cli_subprocess( - [sys.executable, str(script)], - timeout=1.0, - capture_output=True, - text=True, + # Wait briefly for reaping + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"process group members still alive: {alive}" + + def test_run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): + import evals.drivers as drivers_mod + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", ) - elapsed = time.monotonic() - t0 - assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" - assert getattr(ei.value, "killed_process_group", False) is True - - # Wait briefly for reaping - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): - break - time.sleep(0.05) - assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"process group members still alive: {alive}" + + real_comm = subprocess.Popen.communicate + calls = {"n": 0} + + def boom_communicate(self, *a, **k): + calls["n"] += 1 + if calls["n"] == 1: + # Wait until pidfile is written so we can assert both die. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: + break + time.sleep(0.02) + raise KeyboardInterrupt("injected mid-communicate") + return real_comm(self, *a, **k) + + monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_cli_subprocess( + [sys.executable, str(script)], + timeout=30.0, + capture_output=True, + text=True, + ) + assert time.monotonic() - t0 < 6.0 + + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2 + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"group survived BaseException path: {alive}" + # silence unused import lint if any + assert drivers_mod.run_cli_subprocess is run_cli_subprocess + + _d0 = tmp_path / "test_run_cli_subprocess_kills_process_group_on_timeout" + _d0.mkdir() + test_run_cli_subprocess_kills_process_group_on_timeout(_d0) + _d1 = tmp_path / "test_run_cli_subprocess_baseexception_kills_group" + _d1.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_cli_subprocess_baseexception_kills_group(_d1, mp) def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): @@ -171,102 +234,141 @@ def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): pass -def test_run_cli_subprocess_baseexception_kills_group(tmp_path: Path, monkeypatch): - """Non-TimeoutExpired exceptions mid-communicate must still kill the process group.""" - import evals.drivers as drivers_mod +def test_cli_behaviours(tmp_path, monkeypatch): + def test_cli_driver_timeout_notes_process_group_kill(tmp_path): + script = tmp_path / "slow.py" + script.write_text( + textwrap.dedent( + """ + import time + time.sleep(9999) + """ + ), + encoding="utf-8", + ) - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky.py" - script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - pidfile = Path({str(pidfile)!r}) - child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) - pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") - time.sleep(9999) - """ - ), - encoding="utf-8", - ) + # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. + from evals.drivers import run_cli_subprocess as real_runner - real_comm = subprocess.Popen.communicate - calls = {"n": 0} + def short_timeout_runner(cmd, **kwargs): + kwargs = dict(kwargs) + kwargs["timeout"] = 0.5 + # Replace the CLI binary with our sticky sleeper + return real_runner([sys.executable, str(script)], **kwargs) - def boom_communicate(self, *a, **k): - calls["n"] += 1 - if calls["n"] == 1: - # Wait until pidfile is written so we can assert both die. - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: - break - time.sleep(0.02) - raise KeyboardInterrupt("injected mid-communicate") - return real_comm(self, *a, **k) - - monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) - - t0 = time.monotonic() - with pytest.raises(KeyboardInterrupt): - run_cli_subprocess( - [sys.executable, str(script)], - timeout=30.0, - capture_output=True, - text=True, + driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) + t0 = time.monotonic() + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, ) - assert time.monotonic() - t0 < 6.0 - - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): - break - time.sleep(0.05) - assert len(pids) == 2 - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"group survived BaseException path: {alive}" - # silence unused import lint if any - assert drivers_mod.run_cli_subprocess is run_cli_subprocess - - -def test_cli_driver_timeout_notes_process_group_kill(tmp_path: Path): - """ClaudeCliDriver timeout path records timeout_killed_process_group note.""" - script = tmp_path / "slow.py" - script.write_text( - textwrap.dedent( - """ - import time - time.sleep(9999) - """ - ), - encoding="utf-8", - ) + assert time.monotonic() - t0 < 6.0 + assert run.stopped_reason == "timeout" + assert "timeout_killed_process_group" in run.notes + + def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path, monkeypatch): + clock = {"now": 0.0} + monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) + + class MinimalCliDriver(CliDriver): + name = "minimal-cli" + temp_dir_prefix = "plane-eval-minimal-" + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir, child_env + # Harness-owned setup takes five seconds on the fake clock. The + # persisted wall time must start after this hook returns. + clock["now"] = 5.0 + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del model, max_turns, system, launch + return ["minimal", prompt] + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del proc, task_cwd, max_turns, notes + return CliOutput( + final_text="done", + calls=[ + {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, + {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, + ], + ) - # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. - from evals.drivers import run_cli_subprocess as real_runner - - def short_timeout_runner(cmd, **kwargs): - kwargs = dict(kwargs) - kwargs["timeout"] = 0.5 - # Replace the CLI binary with our sticky sleeper - return real_runner([sys.executable, str(script)], **kwargs) - - driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) - t0 = time.monotonic() - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert time.monotonic() - t0 < 6.0 - assert run.stopped_reason == "timeout" - assert "timeout_killed_process_group" in run.notes + def write_complete_sidecar(path: Path, tool: str) -> None: + rows = [ + { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + }, + {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + success_driver: MinimalCliDriver + + def success_runner(cmd, **kwargs): + write_complete_sidecar(success_driver.sidecar_path, "proxy_first") + clock["now"] = 7.0 + return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") + + success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) + success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert success.call_source == "proxy" + assert [call["tool"] for call in success.calls] == ["proxy_first"] + assert success.wall_time_s == 2.0 + + timeout_driver: MinimalCliDriver + + def timeout_runner(cmd, **kwargs): + write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") + clock["now"] = 8.0 + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) + + timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) + timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert timed_out.stopped_reason == "timeout" + assert timed_out.call_source == "proxy" + assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] + assert timed_out.wall_time_s == 3.0 + + _d0 = tmp_path / "test_cli_driver_timeout_notes_process_group_kill" + _d0.mkdir() + test_cli_driver_timeout_notes_process_group_kill(_d0) + _d1 = tmp_path / "test_cli_driver_template_inherits_proxy_first_and_timeout_harvest" + _d1.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(_d1, mp) def test_old_payload_free_sidecar_still_parses(tmp_path: Path): @@ -294,138 +396,82 @@ def test_old_payload_free_sidecar_still_parses(tmp_path: Path): assert "result_text" not in calls[0] -def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path: Path): - side = tmp_path / "s.jsonl" - side.write_text( - json.dumps( - { - "tool": "find_work_items", - "args": {"q": "x"}, - "is_error": False, - "result_chars": 12, - "duration_ms": 5, - "seq": 1, - } +def test_apply_behaviours(tmp_path): + def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path): + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps( + { + "tool": "find_work_items", + "args": {"q": "x"}, + "is_error": False, + "result_chars": 12, + "duration_ms": 5, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", ) - + "\n", - encoding="utf-8", - ) - notes: list[str] = [] - calls, client, src = apply_proxy_sidecar( - [{"tool": "old", "args": {}, "origin": "plane"}], - [], - side, - notes, - ) - assert src == "proxy" - assert calls[0]["tool"] == "find_work_items" - assert calls[0]["duration_ms"] == 5 - assert any("calls_from_proxy" in n for n in notes) - - -def test_apply_proxy_sidecar_empty_fallback(tmp_path: Path): - side = tmp_path / "empty.jsonl" - side.write_text("", encoding="utf-8") - notes: list[str] = [] - original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] - calls, _client, src = apply_proxy_sidecar(original, [], side, notes) - assert calls is original or calls == original - assert "proxy_sidecar_empty" in notes - assert src != "proxy" or calls == original - - -def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path: Path, monkeypatch): - clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) - - class MinimalCliDriver(CliDriver): - name = "minimal-cli" - temp_dir_prefix = "plane-eval-minimal-" - - def write_mcp_config( - self, - temp_dir: Path, - *, - task_cwd: Path, - server_command: list[str], - child_env: dict[str, str], - ) -> CliLaunch: - del temp_dir, child_env - # Harness-owned setup takes five seconds on the fake clock. The - # persisted wall time must start after this hook returns. - clock["now"] = 5.0 - self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) - return CliLaunch(cwd=task_cwd) - - def build_command( - self, - prompt: str, - *, - model: str | None, - max_turns: int, - system: str | None, - launch: CliLaunch, - ) -> list[str]: - del model, max_turns, system, launch - return ["minimal", prompt] - - def parse_output( - self, - proc: subprocess.CompletedProcess[str], - *, - task_cwd: Path, - max_turns: int, - notes: list[str], - ) -> CliOutput: - del proc, task_cwd, max_turns, notes - return CliOutput( - final_text="done", - calls=[ - {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, - {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, - ], + notes: list[str] = [] + calls, client, src = apply_proxy_sidecar( + [{"tool": "old", "args": {}, "origin": "plane"}], + [], + side, + notes, + ) + assert src == "proxy" + assert calls[0]["tool"] == "find_work_items" + assert calls[0]["duration_ms"] == 5 + assert any("calls_from_proxy" in n for n in notes) + + def test_apply_proxy_sidecar_empty_fallback(tmp_path): + side = tmp_path / "empty.jsonl" + side.write_text("", encoding="utf-8") + notes: list[str] = [] + original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] + calls, _client, src = apply_proxy_sidecar(original, [], side, notes) + assert calls is original or calls == original + assert "proxy_sidecar_empty" in notes + assert src != "proxy" or calls == original + + def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path): + p = tmp_path / "s.jsonl" + # Incomplete: one proxy call, no meta. + p.write_text( + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } ) - - def write_complete_sidecar(path: Path, tool: str) -> None: - rows = [ - { - "tool": tool, - "args": {}, - "is_error": False, - "result_chars": 2, - "duration_ms": 1, - "seq": 1, - }, - {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + + "\n", + encoding="utf-8", + ) + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") - - success_driver: MinimalCliDriver - - def success_runner(cmd, **kwargs): - write_complete_sidecar(success_driver.sidecar_path, "proxy_first") - clock["now"] = 7.0 - return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") - - success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) - success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert success.call_source == "proxy" - assert [call["tool"] for call in success.calls] == ["proxy_first"] - assert success.wall_time_s == 2.0 - - timeout_driver: MinimalCliDriver - - def timeout_runner(cmd, **kwargs): - write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") - clock["now"] = 8.0 - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) - - timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) - timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert timed_out.stopped_reason == "timeout" - assert timed_out.call_source == "proxy" - assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] - assert timed_out.wall_time_s == 3.0 + notes: list[str] = [] + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in calls] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + _d0 = tmp_path / "test_apply_proxy_sidecar_replaces_when_nonempty" + _d0.mkdir() + test_apply_proxy_sidecar_replaces_when_nonempty(_d0) + _d1 = tmp_path / "test_apply_proxy_sidecar_empty_fallback" + _d1.mkdir() + test_apply_proxy_sidecar_empty_fallback(_d1) + _d2 = tmp_path / "test_apply_proxy_incomplete_defers_to_richer_cli" + _d2.mkdir() + test_apply_proxy_incomplete_defers_to_richer_cli(_d2) def test_proxy_wrap_server_command(): @@ -447,73 +493,51 @@ def test_proxy_wrap_server_command(): assert with_payloads[5:7] == ["--record-result-payloads", "--"] -def test_load_proxy_sidecar_sorts_by_seq(tmp_path: Path): - p = tmp_path / "s.jsonl" - # Append in reverse response order. - rows = [ - {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, - {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, - { - "row_type": "proxy_meta", - "relayed_lines": 2, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - }, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - calls = load_proxy_sidecar_calls(p) - assert [c["tool"] for c in calls] == ["a", "b"] - - -def test_load_proxy_sidecar_torn_final_line(tmp_path: Path): - p = tmp_path / "s.jsonl" - good = { - "tool": "a", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - # Complete call row + torn final line (no proxy_meta). - p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") - calls, status = load_proxy_sidecar(p) - assert status["state"] == "incomplete" - assert status["torn_line"] is True - assert status["meta"] is None - assert [c["tool"] for c in calls] == ["a"] - - -def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path: Path): - p = tmp_path / "s.jsonl" - # Incomplete: one proxy call, no meta. - p.write_text( - json.dumps( +def test_load_behaviours(tmp_path): + def test_load_proxy_sidecar_sorts_by_seq(tmp_path): + p = tmp_path / "s.jsonl" + # Append in reverse response order. + rows = [ + {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, + {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, { - "tool": "from_proxy", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - ) - + "\n", - encoding="utf-8", - ) - cli = [ - {"tool": "c1", "args": {}, "origin": "plane"}, - {"tool": "c2", "args": {}, "origin": "plane"}, - ] - notes: list[str] = [] - calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) - assert src != "proxy" - assert [c["tool"] for c in calls] == ["c1", "c2"] - assert any("proxy_sidecar_incomplete" in n for n in notes) - assert any("deferred_to_cli" in n for n in notes) + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls = load_proxy_sidecar_calls(p) + assert [c["tool"] for c in calls] == ["a", "b"] + + def test_load_proxy_sidecar_torn_final_line(tmp_path): + p = tmp_path / "s.jsonl" + good = { + "tool": "a", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + # Complete call row + torn final line (no proxy_meta). + p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status["torn_line"] is True + assert status["meta"] is None + assert [c["tool"] for c in calls] == ["a"] + + _d0 = tmp_path / "test_load_proxy_sidecar_sorts_by_seq" + _d0.mkdir() + test_load_proxy_sidecar_sorts_by_seq(_d0) + _d1 = tmp_path / "test_load_proxy_sidecar_torn_final_line" + _d1.mkdir() + test_load_proxy_sidecar_torn_final_line(_d1) def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): @@ -615,18 +639,66 @@ def test_ensure_proxy_pythonpath_injects_repo(): assert env2["PYTHONPATH"].count(str(REPO)) == 1 -def test_timeout_harvests_sidecar_calls(tmp_path: Path): - """Claude timeout path must include sidecar calls made before the timeout.""" - side_calls = [ - { - "tool": "pre_timeout", - "args": {"a": 1}, +def test_timeout_behaviours(tmp_path): + def test_timeout_harvests_sidecar_calls(tmp_path): + side_calls = [ + { + "tool": "pre_timeout", + "args": {"a": 1}, + "is_error": False, + "result_chars": 3, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + + def fake_run(cmd, **kwargs): + # Plant a complete sidecar next to the mcp config (temp dir still alive). + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + # Sidecar path is in the same temp dir as mcp.json for Claude. + # Find sidecar from proxy args in mcp config. + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + log_idx = args.index("--log") + 1 + side = Path(args[log_idx]) + side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "pre_timeout" + + def test_timeout_harvest_waits_for_delayed_meta(tmp_path): + import threading + import time as time_mod + + call_row = { + "tool": "late_meta_tool", + "args": {"n": 1}, "is_error": False, - "result_chars": 3, + "result_chars": 2, "duration_ms": 1, "seq": 1, - }, - { + } + meta_row = { "row_type": "proxy_meta", "relayed_lines": 1, "unparsed_lines": 0, @@ -634,95 +706,52 @@ def test_timeout_harvests_sidecar_calls(tmp_path: Path): "notifications": 0, "pending_left": 0, "child_killed": False, - }, - ] - - def fake_run(cmd, **kwargs): - # Plant a complete sidecar next to the mcp config (temp dir still alive). - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - # Sidecar path is in the same temp dir as mcp.json for Claude. - # Find sidecar from proxy args in mcp config. - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - log_idx = args.index("--log") + 1 - side = Path(args[log_idx]) - side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "pre_timeout" - - -def test_timeout_harvest_waits_for_delayed_meta(tmp_path: Path): - """After CLI kill, harvest must poll until proxy_meta appears (not read early).""" - import threading - import time as time_mod - - call_row = { - "tool": "late_meta_tool", - "args": {"n": 1}, - "is_error": False, - "result_chars": 2, - "duration_ms": 1, - "seq": 1, - } - meta_row = { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - "pumps_alive": False, - } - seen: dict = {"waited": False} + "pumps_alive": False, + } + seen: dict = {"waited": False} - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - # Call row first — no meta yet (simulates proxy still finalizing). - side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") - - def write_meta_later() -> None: - time_mod.sleep(0.45) - with side.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(meta_row) + "\n") - seen["waited"] = True - - threading.Thread(target=write_meta_later, daemon=True).start() - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - t0 = time_mod.monotonic() - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - elapsed = time_mod.monotonic() - t0 - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "late_meta_tool" - assert seen["waited"] is True - # Must have waited for the delayed meta (~0.45s), not returned instantly. - assert elapsed >= 0.4 - assert "proxy_meta_wait_timeout" not in run.notes + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + # Call row first — no meta yet (simulates proxy still finalizing). + side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") + + def write_meta_later() -> None: + time_mod.sleep(0.45) + with side.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta_row) + "\n") + seen["waited"] = True + + threading.Thread(target=write_meta_later, daemon=True).start() + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + t0 = time_mod.monotonic() + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + elapsed = time_mod.monotonic() - t0 + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "late_meta_tool" + assert seen["waited"] is True + # Must have waited for the delayed meta (~0.45s), not returned instantly. + assert elapsed >= 0.4 + assert "proxy_meta_wait_timeout" not in run.notes + + _d0 = tmp_path / "test_timeout_harvests_sidecar_calls" + _d0.mkdir() + test_timeout_harvests_sidecar_calls(_d0) + _d1 = tmp_path / "test_timeout_harvest_waits_for_delayed_meta" + _d1.mkdir() + test_timeout_harvest_waits_for_delayed_meta(_d1) def test_wait_for_proxy_meta_unit(tmp_path: Path): diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index d690b70d..0cb07fd6 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -254,174 +254,240 @@ def test_normalize_claude_usage_real_shape(): assert total["source"] == "modelUsage" -def test_parse_claude_json_result_usage_and_cost(): - out = parse_claude_json_result(CLAUDE_JSON_RESULT) - assert out["final_text"] == "The work item is in Todo." - assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" - assert out["num_turns"] == 3 - assert out["usage"]["input_tokens"] == 10 - assert out["usage"]["total_cost_usd"] == 0.291 - assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 - assert out["calls"] == [] - assert out["stopped_reason"] == "end_turn" - - -def test_parse_claude_json_with_embedded_calls_splits_toolsearch(): - """F1: ToolSearch is client; only plane MCP tools remain in calls.""" - out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) - assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] - assert all(c["origin"] == "plane" for c in out["calls"]) - assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] - assert out["calls"][0]["args"]["limit"] == 10 - - -def test_parse_claude_transcript_calls(tmp_path: Path): - p = tmp_path / "sess.jsonl" - p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - tagged = parse_claude_transcript_calls(p) - plane, client = split_plane_and_client_calls(tagged) - assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] - assert [c["tool"] for c in client] == ["ToolSearch"] - assert plane[0]["args"]["project_id"] == "proj-1" - - -def test_parse_codex_jsonl_events(): - out = parse_codex_jsonl_events(CODEX_JSONL) - assert out["session_id"] == "sess-codex-1" - assert out["final_text"] == "All set." - assert out["usage"]["input_tokens"] == 5000 - assert out["usage"]["cache_read_input_tokens"] == 1000 - # plane only in calls; exec_command is client machinery - tools = [c["tool"] for c in out["calls"]] - assert tools == ["find_work_items"] - assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] - assert out["stopped_reason"] == "end_turn" - - -def test_parse_codex_jsonl_events_v0147_schema(): - """Parser fixture: exact four-line v0.147 stream (thread_id, PING, usage).""" - out = parse_codex_jsonl_events(CODEX_V0147_JSONL) - assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" - assert out["final_text"] == "PING" - assert out["usage"]["input_tokens"] == 16050 - assert out["usage"]["cache_read_input_tokens"] == 15104 - assert out["usage"]["cache_creation_input_tokens"] == 0 - assert out["usage"]["output_tokens"] == 5 - - -def test_parse_codex_jsonl_events_mixed_old_and_new_schema(): - """Single parser: new keys + legacy keys in one stream both contribute.""" - mixed = "\n".join( - [ - json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), - json.dumps( +def test_parse_behaviours(tmp_path): + def test_parse_claude_behaviours(): + def test_parse_claude_json_result_usage_and_cost(): + out = parse_claude_json_result(CLAUDE_JSON_RESULT) + assert out["final_text"] == "The work item is in Todo." + assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + assert out["num_turns"] == 3 + assert out["usage"]["input_tokens"] == 10 + assert out["usage"]["total_cost_usd"] == 0.291 + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["calls"] == [] + assert out["stopped_reason"] == "end_turn" + + def test_parse_claude_json_with_embedded_calls_splits_toolsearch(): + out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) + assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] + assert all(c["origin"] == "plane" for c in out["calls"]) + assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] + assert out["calls"][0]["args"]["limit"] == 10 + + def test_parse_claude_json_preserves_error_subtype(): + out = parse_claude_json_result( { - "type": "item.completed", - "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, - } - ), - # Legacy call row still harvested - json.dumps( - { - "type": "response_item", - "payload": { - "type": "function_call", - "name": "mcp__plane__list_work_items", - "arguments": json.dumps({"project_id": "p"}), - }, - } - ), - json.dumps( - { - "type": "turn.completed", - "usage": { - "input_tokens": 10, - "cached_input_tokens": 0, - "cache_write_input_tokens": 0, - "output_tokens": 2, - }, + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "x", + "session_id": "s", + "num_turns": 1, } - ), - ] - ) - out = parse_codex_jsonl_events(mixed) - assert out["session_id"] == "thread-new-1" - assert "Hello from new" in out["final_text"] - assert [c["tool"] for c in out["calls"]] == ["list_work_items"] - assert out["usage"]["input_tokens"] == 10 - - -def test_find_codex_rollout_exact_match_and_unmatched(tmp_path: Path, monkeypatch): - """Exact session id match only; no newest-after-ts substitution.""" - from evals import drivers as drivers_mod - - sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" - sessions.mkdir(parents=True) - tid = "019ff6af-69df-7022-b353-322ffe1ececb" - # Unrelated newer session (must never be returned when looking for tid) - other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" - other.write_text( - json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", - encoding="utf-8", - ) - # Exact match via filename suffix - match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" - match.write_text( - json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", - encoding="utf-8", - ) - - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout(tid) - assert found is not None - assert tid in found.name - # Must not return the other concurrent session - assert "other-session" not in found.name - - assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None - assert drivers_mod.find_codex_rollout(None) is None - - -def test_find_codex_rollout_session_meta_id(tmp_path: Path, monkeypatch): - from evals import drivers as drivers_mod - - sessions = tmp_path / ".codex" / "sessions" - sessions.mkdir(parents=True) - p = sessions / "rollout-meta-only.jsonl" - p.write_text( - json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", - encoding="utf-8", - ) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout("sess-meta-42") - assert found is not None - assert found.name == "rollout-meta-only.jsonl" - - -def test_codex_driver_notes_rollout_unmatched_when_no_file(tmp_path: Path, monkeypatch): - """When thread_id is known but no rollout file matches, note codex_rollout_unmatched.""" - - # Empty sessions dir under fake home - (tmp_path / ".codex" / "sessions").mkdir(parents=True) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") + ) + assert out["stopped_reason"] == "error_during_execution" + + test_parse_claude_json_result_usage_and_cost() + test_parse_claude_json_with_embedded_calls_splits_toolsearch() + test_parse_claude_json_preserves_error_subtype() + + def test_parse_claude_transcript_calls(tmp_path): + p = tmp_path / "sess.jsonl" + p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + tagged = parse_claude_transcript_calls(p) + plane, client = split_plane_and_client_calls(tagged) + assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in client] == ["ToolSearch"] + assert plane[0]["args"]["project_id"] == "proj-1" + + def test_parse_codex_behaviours(): + def test_parse_codex_jsonl_events(): + out = parse_codex_jsonl_events(CODEX_JSONL) + assert out["session_id"] == "sess-codex-1" + assert out["final_text"] == "All set." + assert out["usage"]["input_tokens"] == 5000 + assert out["usage"]["cache_read_input_tokens"] == 1000 + # plane only in calls; exec_command is client machinery + tools = [c["tool"] for c in out["calls"]] + assert tools == ["find_work_items"] + assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] + assert out["stopped_reason"] == "end_turn" + + def test_parse_codex_jsonl_events_v0147_schema(): + out = parse_codex_jsonl_events(CODEX_V0147_JSONL) + assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" + assert out["final_text"] == "PING" + assert out["usage"]["input_tokens"] == 16050 + assert out["usage"]["cache_read_input_tokens"] == 15104 + assert out["usage"]["cache_creation_input_tokens"] == 0 + assert out["usage"]["output_tokens"] == 5 + + def test_parse_codex_jsonl_events_mixed_old_and_new_schema(): + mixed = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, + } + ), + # Legacy call row still harvested + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__list_work_items", + "arguments": json.dumps({"project_id": "p"}), + }, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + }, + } + ), + ] + ) + out = parse_codex_jsonl_events(mixed) + assert out["session_id"] == "thread-new-1" + assert "Hello from new" in out["final_text"] + assert [c["tool"] for c in out["calls"]] == ["list_work_items"] + assert out["usage"]["input_tokens"] == 10 + + test_parse_codex_jsonl_events() + test_parse_codex_jsonl_events_v0147_schema() + test_parse_codex_jsonl_events_mixed_old_and_new_schema() + + test_parse_claude_behaviours() + _d1 = tmp_path / "test_parse_claude_transcript_calls" + _d1.mkdir() + test_parse_claude_transcript_calls(_d1) + test_parse_codex_behaviours() + + +def test_find_behaviours(tmp_path, monkeypatch): + def test_find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" + sessions.mkdir(parents=True) + tid = "019ff6af-69df-7022-b353-322ffe1ececb" + # Unrelated newer session (must never be returned when looking for tid) + other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" + other.write_text( + json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", + encoding="utf-8", + ) + # Exact match via filename suffix + match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" + match.write_text( + json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", + encoding="utf-8", + ) - driver = CodexCliDriver(runner=fake_run, use_proxy=False) - run = driver.run_task( - "ping", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - # Final text still from stdout (new schema) — never from a wrong rollout - assert run.final_text == "PING" - # Unmatched note only when looking for enrichment; with final_text present - # need_rollout is false for final_text — still may note if no calls. - # v0147 fixture has no tool calls → need_rollout True → unmatched note. - assert "codex_rollout_unmatched" in run.notes + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout(tid) + assert found is not None + assert tid in found.name + # Must not return the other concurrent session + assert "other-session" not in found.name + + assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None + assert drivers_mod.find_codex_rollout(None) is None + + def test_find_codex_rollout_session_meta_id(tmp_path, monkeypatch): + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" + sessions.mkdir(parents=True) + p = sessions / "rollout-meta-only.jsonl" + p.write_text( + json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout("sess-meta-42") + assert found is not None + assert found.name == "rollout-meta-only.jsonl" + + _d0 = tmp_path / "test_find_codex_rollout_exact_match_and_unmatched" + _d0.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_find_codex_rollout_exact_match_and_unmatched(_d0, mp) + _d1 = tmp_path / "test_find_codex_rollout_session_meta_id" + _d1.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_find_codex_rollout_session_meta_id(_d1, mp) + + +def test_codex_behaviours(tmp_path, monkeypatch): + def test_codex_driver_notes_rollout_unmatched_when_no_file(tmp_path, monkeypatch): + (tmp_path / ".codex" / "sessions").mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run, use_proxy=False) + run = driver.run_task( + "ping", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + # Final text still from stdout (new schema) — never from a wrong rollout + assert run.final_text == "PING" + # Unmatched note only when looking for enrichment; with final_text present + # need_rollout is false for final_text — still may note if no calls. + # v0147 fixture has no tool calls → need_rollout True → unmatched note. + assert "codex_rollout_unmatched" in run.notes + + def test_codex_driver_behaviours(): + def test_codex_driver_parses_fake_stdout_no_live(): + def fake_run(cmd, **kwargs): + assert cmd[0] == "codex" + assert "exec" in cmd + assert "--json" in cmd + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="gpt-test", + max_turns=5, + cwd=Path("/tmp"), + ) + assert run.experimental is True + assert run.call_source == "stream" + assert run.calls[0]["tool"] == "find_work_items" + assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] + assert run.usage is not None + assert run.usage["input_tokens"] == 5000 + assert run.final_text == "All set." + + def test_codex_driver_refuses_live_by_default(): + driver = CodexCliDriver() # real subprocess.run + with pytest.raises(RuntimeError, match="refuses live"): + driver.run_task("x", mcp_env={}, model=None, max_turns=1) + + test_codex_driver_parses_fake_stdout_no_live() + test_codex_driver_refuses_live_by_default() + + _d0 = tmp_path / "test_codex_driver_notes_rollout_unmatched_when_no_file" + _d0.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_codex_driver_notes_rollout_unmatched_when_no_file(_d0, mp) + test_codex_driver_behaviours() def test_max_turns_detection_from_num_turns(): @@ -449,166 +515,305 @@ def fake_run(cmd, **kwargs): assert run.usage_total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 -def test_claude_driver_falls_back_to_transcript(tmp_path: Path, monkeypatch): - session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" - payload = { - **CLAUDE_JSON_RESULT, - "session_id": session_id, - "tool_calls": [], # force transcript path - "result": "from-json", - } - - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") - - # Plant transcript where find_claude_transcript looks - munged = str(tmp_path.resolve()).replace("/", "-") - proj = Path.home() / ".claude" / "projects" / munged - proj.mkdir(parents=True, exist_ok=True) - transcript = proj / f"{session_id}.jsonl" - transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - monkeypatch.setenv("HOME", str(Path.home())) # keep real home for this test path - - driver = ClaudeCliDriver(runner=fake_run) - run = driver.run_task( - "prompt", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=10, - cwd=tmp_path, - ) - assert run.call_source == "transcript" - assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] - assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] - assert run.final_text == "from-json" - # cleanup planted file - transcript.unlink(missing_ok=True) +def test_claude_behaviours(tmp_path, monkeypatch): + def test_claude_driver_falls_back_to_transcript(tmp_path, monkeypatch): + session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], # force transcript path + "result": "from-json", + } + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") -def test_claude_driver_writes_mcp_config_and_cmd_flags(tmp_path: Path): - seen: dict[str, Any] = {} + # Plant transcript where find_claude_transcript looks + munged = str(tmp_path.resolve()).replace("/", "-") + proj = Path.home() / ".claude" / "projects" / munged + proj.mkdir(parents=True, exist_ok=True) + transcript = proj / f"{session_id}.jsonl" + transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + monkeypatch.setenv("HOME", str(Path.home())) # keep real home for this test path - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - seen["cwd"] = kwargs.get("cwd") - # Return minimal valid JSON - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), - stderr="", + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, ) - - driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") - driver.run_task( - "hello", - mcp_env={ - "PLANE_API_KEY": "key", - "PLANE_WORKSPACE_SLUG": "slug", - "PLANE_BASE_URL": "https://api.example", - "CUSTOM_SETTING": "enabled", - "PATH": "/usr/bin", - }, - model="sonnet", - max_turns=7, - cwd=tmp_path, - system="sys", - ) - cmd = seen["cmd"] - assert cmd[0] == "claude" - assert "-p" in cmd - assert "--output-format" in cmd and "json" in cmd - assert "--mcp-config" in cmd - assert "--max-turns" in cmd and "7" in cmd - assert "--model" in cmd and "sonnet" in cmd - assert "--permission-mode" in cmd and "bypassPermissions" in cmd - assert "--strict-mcp-config" in cmd - # mcp-config path is a temp file cleaned after run — re-check via write helper - cfg = tmp_path / "mcp.json" - write_claude_mcp_config( - cfg, - command="/venv/bin/python", - args=["-m", "plane_mcp", "stdio"], - env={"PLANE_API_KEY": "key"}, - ) - data = json.loads(cfg.read_text()) - assert "mcpServers" in data - assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] - - -def test_claude_driver_server_command_override(tmp_path: Path): - """External surfaces: --server-cmd replaces the default `-m plane_mcp stdio` launch.""" - seen: dict[str, Any] = {} - - def fake_run(cmd, **kwargs): - # Capture the mcp.json content while it still exists (temp dir). - cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp_cfg"] = json.loads(cfg_path.read_text()) - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), - stderr="", + assert run.call_source == "transcript" + assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] + assert run.final_text == "from-json" + # cleanup planted file + transcript.unlink(missing_ok=True) + + def test_claude_driver_writes_mcp_config_and_cmd_flags(tmp_path): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + seen["cwd"] = kwargs.get("cwd") + # Return minimal valid JSON + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") + driver.run_task( + "hello", + mcp_env={ + "PLANE_API_KEY": "key", + "PLANE_WORKSPACE_SLUG": "slug", + "PLANE_BASE_URL": "https://api.example", + "CUSTOM_SETTING": "enabled", + "PATH": "/usr/bin", + }, + model="sonnet", + max_turns=7, + cwd=tmp_path, + system="sys", ) + cmd = seen["cmd"] + assert cmd[0] == "claude" + assert "-p" in cmd + assert "--output-format" in cmd and "json" in cmd + assert "--mcp-config" in cmd + assert "--max-turns" in cmd and "7" in cmd + assert "--model" in cmd and "sonnet" in cmd + assert "--permission-mode" in cmd and "bypassPermissions" in cmd + assert "--strict-mcp-config" in cmd + # mcp-config path is a temp file cleaned after run — re-check via write helper + cfg = tmp_path / "mcp.json" + write_claude_mcp_config( + cfg, + command="/venv/bin/python", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "key"}, + ) + data = json.loads(cfg.read_text()) + assert "mcpServers" in data + assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] + + def test_claude_driver_server_command_override(tmp_path): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + # Capture the mcp.json content while it still exists (temp dir). + cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp_cfg"] = json.loads(cfg_path.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver( + runner=fake_run, + server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], + ) + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp_cfg"]["mcpServers"]["plane"] + # Default use_proxy=True: command is the proxy; real server follows "--". + assert server["args"][:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + dash = server["args"].index("--") + assert server["args"][dash + 1 :] == [ + "/elsewhere/.venv/bin/plane-mcp-server", + "stdio", + "--mode", + "candidate", + ] + # Explicit foreign selection variables pass through to the child. + assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" + + def test_claude_driver_behaviours(): + def test_claude_driver_timeout_returns_agent_run_not_raise(): + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=2, + cwd=Path("/tmp"), + ) + assert run.stopped_reason == "timeout" + assert run.calls == [] + assert any("timeout after" in n for n in run.notes) + + def test_claude_driver_json_parse_failure_raises_for_infra_cli(): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") + + driver = ClaudeCliDriver(runner=fake_run) + with pytest.raises(RuntimeError, match="claude cli failed"): + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=Path("/tmp"), + ) + + test_claude_driver_timeout_returns_agent_run_not_raise() + test_claude_driver_json_parse_failure_raises_for_infra_cli() + + def test_claude_driver_uses_proxy_in_mcp_config(tmp_path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + # Leave empty sidecar (proxy not really run under fake runner). + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) - driver = ClaudeCliDriver( - runner=fake_run, - server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], - ) - driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, - model="sonnet", - max_turns=3, - cwd=tmp_path, - ) - server = seen["mcp_cfg"]["mcpServers"]["plane"] - # Default use_proxy=True: command is the proxy; real server follows "--". - assert server["args"][:3] == ["-m", "evals.proxy", "--log"] - assert "--" in server["args"] - dash = server["args"].index("--") - assert server["args"][dash + 1 :] == [ - "/elsewhere/.venv/bin/plane-mcp-server", - "stdio", - "--mode", - "candidate", - ] - # Explicit foreign selection variables pass through to the child. - assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" - - -def test_codex_driver_parses_fake_stdout_no_live(): - def fake_run(cmd, **kwargs): - assert cmd[0] == "codex" - assert "exec" in cmd - assert "--json" in cmd - return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") - - driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed - run = driver.run_task( - "do it", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="gpt-test", - max_turns=5, - cwd=Path("/tmp"), - ) - assert run.experimental is True - assert run.call_source == "stream" - assert run.calls[0]["tool"] == "find_work_items" - assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] - assert run.usage is not None - assert run.usage["input_tokens"] == 5000 - assert run.final_text == "All set." - - -def test_codex_driver_refuses_live_by_default(): - driver = CodexCliDriver() # real subprocess.run - with pytest.raises(RuntimeError, match="refuses live"): - driver.run_task("x", mcp_env={}, model=None, max_turns=1) + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["command"] == "/venv/bin/python" + assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + assert "plane_mcp" in server["args"] + assert "proxy_sidecar_empty" in run.notes + + def test_claude_driver_proxy_disabled_no_wrap(tmp_path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["args"] == ["-m", "plane_mcp", "stdio"] + + def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) -def test_known_drivers(): - assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert str(REPO) in seen["env"].get("PYTHONPATH", "") + + _d0 = tmp_path / "test_claude_driver_falls_back_to_transcript" + _d0.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_claude_driver_falls_back_to_transcript(_d0, mp) + _d1 = tmp_path / "test_claude_driver_writes_mcp_config_and_cmd_flags" + _d1.mkdir() + test_claude_driver_writes_mcp_config_and_cmd_flags(_d1) + _d2 = tmp_path / "test_claude_driver_server_command_override" + _d2.mkdir() + test_claude_driver_server_command_override(_d2) + test_claude_driver_behaviours() + _d4 = tmp_path / "test_claude_driver_uses_proxy_in_mcp_config" + _d4.mkdir() + test_claude_driver_uses_proxy_in_mcp_config(_d4) + _d5 = tmp_path / "test_claude_driver_proxy_disabled_no_wrap" + _d5.mkdir() + test_claude_driver_proxy_disabled_no_wrap(_d5) + _d6 = tmp_path / "test_claude_mcp_env_has_pythonpath_when_proxied" + _d6.mkdir() + test_claude_mcp_env_has_pythonpath_when_proxied(_d6) + + +def test_known_drivers_behaviours(): + def test_known_drivers(): + assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + + def test_known_drivers_and_get_driver(): + assert "antigravity-cli" in KNOWN_DRIVERS + assert "opencode-cli" in KNOWN_DRIVERS + assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) + assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) + + test_known_drivers() + test_known_drivers_and_get_driver() def test_get_driver_api(): @@ -617,166 +822,122 @@ def test_get_driver_api(): assert isinstance(get_driver("codex-cli"), CodexCliDriver) -def test_parse_claude_json_preserves_error_subtype(): - out = parse_claude_json_result( - { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": "x", - "session_id": "s", - "num_turns": 1, +def test_antigravity_behaviours(tmp_path): + def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + env = kwargs.get("env") or {} + seen["env"] = env + home = env.get("HOME") + if home: + cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" + seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, + model="gemini-2.5", + max_turns=5, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "agy" + assert "-p" in seen["cmd"] + assert "--output-format" in seen["cmd"] + assert "json" in seen["cmd"] + assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] + assert "no_turn_cap" in run.notes + assert seen.get("mcp_cfg") is not None + assert "mcpServers" in seen["mcp_cfg"] + assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) + + def test_antigravity_fallback_runner_timeout_harvests(tmp_path): + call_row = { + "tool": "g_tool", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, } - ) - assert out["stopped_reason"] == "error_during_execution" - - -def test_claude_driver_timeout_returns_agent_run_not_raise(): - def fake_run(cmd, **kwargs): - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) - - driver = ClaudeCliDriver(runner=fake_run) - run = driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=2, - cwd=Path("/tmp"), - ) - assert run.stopped_reason == "timeout" - assert run.calls == [] - assert any("timeout after" in n for n in run.notes) - - -def test_claude_driver_json_parse_failure_raises_for_infra_cli(): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") - driver = ClaudeCliDriver(runner=fake_run) - with pytest.raises(RuntimeError, match="claude cli failed"): - driver.run_task( - "hello", + def fake_run(cmd, **kwargs): + run_env = kwargs.get("env") or {} + home = run_env.get("HOME") + if home: + # First attempt includes env= — plant sidecar from dual-written mcp config, + # then reject env so the driver retries without it. + for rel in ( + Path(home) / ".gemini" / "config" / "mcp_config.json", + Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + cfg = json.loads(rel.read_text()) + args = cfg["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text( + "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", + encoding="utf-8", + ) + break + raise TypeError("runner does not accept env=") + # Fallback call (no env) times out — outer except must still harvest. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, model=None, max_turns=1, - cwd=Path("/tmp"), - ) - - -def test_claude_driver_uses_proxy_in_mcp_config(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - # Leave empty sidecar (proxy not really run under fake runner). - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=3, - cwd=tmp_path, - ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["command"] == "/venv/bin/python" - assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] - assert "--" in server["args"] - assert "plane_mcp" in server["args"] - assert "proxy_sidecar_empty" in run.notes - - -def test_claude_driver_proxy_disabled_no_wrap(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", + cwd=tmp_path, ) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") - driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["args"] == ["-m", "plane_mcp", "stdio"] - - -def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - env = kwargs.get("env") or {} - seen["env"] = env - home = env.get("HOME") - if home: - cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" - seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None - return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "do it", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, - model="gemini-2.5", - max_turns=5, - cwd=tmp_path, - ) - assert seen["cmd"][0] == "agy" - assert "-p" in seen["cmd"] - assert "--output-format" in seen["cmd"] - assert "json" in seen["cmd"] - assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] - assert "no_turn_cap" in run.notes - assert seen.get("mcp_cfg") is not None - assert "mcpServers" in seen["mcp_cfg"] - assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) - - -def test_write_antigravity_mcp_config_shape(tmp_path: Path): - p = tmp_path / "mcp_config.json" - write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) - data = json.loads(p.read_text()) - assert data["mcpServers"]["plane"]["command"] == "python" - assert data["mcpServers"]["plane"]["env"]["A"] == "1" + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "g_tool" + + _d0 = tmp_path / "test_antigravity_driver_writes_mcp_config_under_isolated_home" + _d0.mkdir() + test_antigravity_driver_writes_mcp_config_under_isolated_home(_d0) + _d1 = tmp_path / "test_antigravity_fallback_runner_timeout_harvests" + _d1.mkdir() + test_antigravity_fallback_runner_timeout_harvests(_d1) + + +def test_write_behaviours(tmp_path): + def test_write_antigravity_mcp_config_shape(tmp_path): + p = tmp_path / "mcp_config.json" + write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) + data = json.loads(p.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + assert data["mcpServers"]["plane"]["env"]["A"] == "1" + + def test_write_opencode_mcp_config_shape(tmp_path): + p = tmp_path / "opencode.json" + write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) + data = json.loads(p.read_text()) + assert data["mcp"]["plane"]["command"][0] == "py" + assert data["mcp"]["plane"]["environment"]["K"] == "V" + + _d0 = tmp_path / "test_write_antigravity_mcp_config_shape" + _d0.mkdir() + test_write_antigravity_mcp_config_shape(_d0) + _d1 = tmp_path / "test_write_opencode_mcp_config_shape" + _d1.mkdir() + test_write_opencode_mcp_config_shape(_d1) def test_opencode_driver_writes_project_config(tmp_path: Path): @@ -809,21 +970,6 @@ def fake_run(cmd, **kwargs): assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) -def test_write_opencode_mcp_config_shape(tmp_path: Path): - p = tmp_path / "opencode.json" - write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) - data = json.loads(p.read_text()) - assert data["mcp"]["plane"]["command"][0] == "py" - assert data["mcp"]["plane"]["environment"]["K"] == "V" - - -def test_known_drivers_and_get_driver(): - assert "antigravity-cli" in KNOWN_DRIVERS - assert "opencode-cli" in KNOWN_DRIVERS - assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) - assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) - - def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): real_home = tmp_path / "real" cli = real_home / ".gemini" / "antigravity-cli" @@ -860,93 +1006,3 @@ def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): # Real home byte-for-byte untouched (including oauth token). after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} assert after == before - - -def test_antigravity_fallback_runner_timeout_harvests(tmp_path: Path): - """TypeError fallback path's TimeoutExpired must still harvest via wait-for-meta.""" - call_row = { - "tool": "g_tool", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - meta = { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - } - - def fake_run(cmd, **kwargs): - run_env = kwargs.get("env") or {} - home = run_env.get("HOME") - if home: - # First attempt includes env= — plant sidecar from dual-written mcp config, - # then reject env so the driver retries without it. - for rel in ( - Path(home) / ".gemini" / "config" / "mcp_config.json", - Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", - ): - if rel.is_file(): - cfg = json.loads(rel.read_text()) - args = cfg["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - side.write_text( - "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", - encoding="utf-8", - ) - break - raise TypeError("runner does not accept env=") - # Fallback call (no env) times out — outer except must still harvest. - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "g_tool" - - -def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path: Path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - - ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert str(REPO) in seen["env"].get("PYTHONPATH", "") diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py index b246b20f..f70a8ed6 100644 --- a/tests/evals/report/test_compare.py +++ b/tests/evals/report/test_compare.py @@ -12,67 +12,71 @@ ) -def test_sign_test_all_positive_hand_computed(): - """n=5 non-zero, all positive → two-sided p = 2 * (1/32) = 0.0625.""" - deltas = [1.0, 2.0, 3.0, 0.5, 4.0] - p = sign_test_pvalue(deltas) - assert p == pytest.approx(2.0 * (1.0 / 32.0)) - assert p == pytest.approx(0.0625) - - -def test_sign_test_four_of_five_hand_computed(): - """n=5, k=4 positive → right tail (C(5,4)+C(5,5))/32 = 6/32; p=2*6/32=0.375.""" - deltas = [1.0, 1.0, 1.0, 1.0, -1.0] - p = sign_test_pvalue(deltas) - right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 - assert p == pytest.approx(2.0 * right) - assert p == pytest.approx(0.375) - - -def test_sign_test_drops_zeros_and_none_when_empty(): - assert sign_test_pvalue([0.0, 0.0]) is None - assert sign_test_pvalue([]) is None - # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 - assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) - - -def test_ab_compare_paired_deltas_and_sign_test(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, - {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 - {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired - ] - cmp = ab_compare(rows_a, rows_b) - assert cmp["n_paired"] == 2 - deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} - assert deltas["R1"] == -3.0 - assert deltas["R2"] == 1.0 - assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] - assert cmp["sign_test_p"] is not None - assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 - assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 - - -def test_ab_compare_multi_rep_uses_median_successful_call_counts(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, - ] - - cmp = ab_compare(rows_a, rows_b) - - assert cmp["multi_rep"] is True - assert cmp["unstable_a"] == 1 - assert cmp["unstable_b"] == 0 - assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] +def test_sign_test_behaviours(): + def test_sign_test_all_positive_hand_computed(): + deltas = [1.0, 2.0, 3.0, 0.5, 4.0] + p = sign_test_pvalue(deltas) + assert p == pytest.approx(2.0 * (1.0 / 32.0)) + assert p == pytest.approx(0.0625) + + def test_sign_test_four_of_five_hand_computed(): + deltas = [1.0, 1.0, 1.0, 1.0, -1.0] + p = sign_test_pvalue(deltas) + right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 + assert p == pytest.approx(2.0 * right) + assert p == pytest.approx(0.375) + + def test_sign_test_drops_zeros_and_none_when_empty(): + assert sign_test_pvalue([0.0, 0.0]) is None + assert sign_test_pvalue([]) is None + # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 + assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) + + test_sign_test_all_positive_hand_computed() + test_sign_test_four_of_five_hand_computed() + test_sign_test_drops_zeros_and_none_when_empty() + + +def test_ab_compare_behaviours(): + def test_ab_compare_paired_deltas_and_sign_test(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired + ] + cmp = ab_compare(rows_a, rows_b) + assert cmp["n_paired"] == 2 + deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} + assert deltas["R1"] == -3.0 + assert deltas["R2"] == 1.0 + assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] + assert cmp["sign_test_p"] is not None + assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 + assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 + + def test_ab_compare_multi_rep_uses_median_successful_call_counts(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, + ] + + cmp = ab_compare(rows_a, rows_b) + + assert cmp["multi_rep"] is True + assert cmp["unstable_a"] == 1 + assert cmp["unstable_b"] == 0 + assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] + + test_ab_compare_paired_deltas_and_sign_test() + test_ab_compare_multi_rep_uses_median_successful_call_counts() diff --git a/tests/evals/report/test_load.py b/tests/evals/report/test_load.py index 6c33007e..28465b0d 100644 --- a/tests/evals/report/test_load.py +++ b/tests/evals/report/test_load.py @@ -18,55 +18,64 @@ def test_is_infra_error_row_covers_infrastructure_prefix(): assert is_infra_error_row({"error_class": "task"}) is False -def test_load_rows_dedupe_latest_wins(tmp_path: Path): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p) # default dedupe=latest - assert len(loaded) == 1 - assert loaded[0].num_calls == 9 - assert loaded[0].success is False - - -def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path: Path, capsys): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p, dedupe="none") - assert len(loaded) == 2 - err = capsys.readouterr().err - assert "duplicate" in err - assert "R1" in err - - -def test_load_rows_skips_meta_and_missing_task_id(tmp_path: Path): - p = tmp_path / "r.jsonl" - lines = [ - json.dumps( - { - "row_type": "meta", - "run_id": "abc", - "label": "candidate", - "battery": "deadbeef0001", - "model": "sonnet", - "driver": "claude-cli", - "git_sha": "x", - "ts": "t", - } - ), - json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id - json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), - ] - p.write_text("\n".join(lines) + "\n", encoding="utf-8") - rows = load_rows(p) - assert len(rows) == 1 - assert rows[0].task_id == "R1" +def test_load_behaviours(tmp_path, capsys): + def test_load_rows_dedupe_latest_wins(tmp_path): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p) # default dedupe=latest + assert len(loaded) == 1 + assert loaded[0].num_calls == 9 + assert loaded[0].success is False + + def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path, capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p, dedupe="none") + assert len(loaded) == 2 + err = capsys.readouterr().err + assert "duplicate" in err + assert "R1" in err + + def test_load_rows_skips_meta_and_missing_task_id(tmp_path): + p = tmp_path / "r.jsonl" + lines = [ + json.dumps( + { + "row_type": "meta", + "run_id": "abc", + "label": "candidate", + "battery": "deadbeef0001", + "model": "sonnet", + "driver": "claude-cli", + "git_sha": "x", + "ts": "t", + } + ), + json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + rows = load_rows(p) + assert len(rows) == 1 + assert rows[0].task_id == "R1" + + _d0 = tmp_path / "test_load_rows_dedupe_latest_wins" + _d0.mkdir() + test_load_rows_dedupe_latest_wins(_d0) + _d1 = tmp_path / "test_load_rows_no_dedupe_warns_on_duplicate_keys" + _d1.mkdir() + test_load_rows_no_dedupe_warns_on_duplicate_keys(_d1, capsys) + _d2 = tmp_path / "test_load_rows_skips_meta_and_missing_task_id" + _d2.mkdir() + test_load_rows_skips_meta_and_missing_task_id(_d2) def test_real_historical_rows_parse_and_report_with_backward_defaults(): diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index a09b3a31..06eb8d19 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -16,35 +16,61 @@ ) -def test_summarize_excludes_infra_errors_from_success(): - rows = [ - {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "HttpError: 409", - "error_class": "infra_seed", - }, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "timeout after 120s", - "error_class": "infra_cli", - }, - {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, - ] - summary = summarize(rows) - assert summary.infra_errors == 2 - assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows - assert summary.tasks["R1"].k == 1 - assert summary.tasks["R1"].success == "1/2" - assert summary.tasks["R1"].infra_err == 2 - assert is_infra_error_row(rows[1]) is True - assert is_infra_error_row(rows[0]) is False +def test_summarize_behaviours(): + def test_summarize_excludes_infra_errors_from_success(): + rows = [ + {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "HttpError: 409", + "error_class": "infra_seed", + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "timeout after 120s", + "error_class": "infra_cli", + }, + {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, + ] + summary = summarize(rows) + assert summary.infra_errors == 2 + assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows + assert summary.tasks["R1"].k == 1 + assert summary.tasks["R1"].success == "1/2" + assert summary.tasks["R1"].infra_err == 2 + assert is_infra_error_row(rows[1]) is True + assert is_infra_error_row(rows[0]) is False + + def test_summarize_aggregate_wilson_and_call_variance(): + rows = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + ] + s = summarize(rows) + assert s.tasks["R1"].n == 3 + assert s.tasks["R1"].k == 2 + assert s.tasks["R1"].calls_min == 2.0 + assert s.tasks["R1"].calls_max == 6.0 + assert s.tasks["R1"].med_calls == 4.0 + assert s.tasks["R1"].unstable is True + assert s.tasks["R2"].unstable is False + assert s.aggregate_k == 3 + assert s.aggregate_n == 4 + assert s.multi_rep is True + assert s.unstable_task_ids == ["R1"] + assert s.unstable_tasks == 1 + assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 + + test_summarize_excludes_infra_errors_from_success() + test_summarize_aggregate_wilson_and_call_variance() def test_wilson_interval_bounds(): @@ -57,29 +83,6 @@ def test_wilson_interval_bounds(): assert wilson_interval(0, 0) == (0.0, 0.0) -def test_summarize_aggregate_wilson_and_call_variance(): - rows = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - ] - s = summarize(rows) - assert s.tasks["R1"].n == 3 - assert s.tasks["R1"].k == 2 - assert s.tasks["R1"].calls_min == 2.0 - assert s.tasks["R1"].calls_max == 6.0 - assert s.tasks["R1"].med_calls == 4.0 - assert s.tasks["R1"].unstable is True - assert s.tasks["R2"].unstable is False - assert s.aggregate_k == 3 - assert s.aggregate_n == 4 - assert s.multi_rep is True - assert s.unstable_task_ids == ["R1"] - assert s.unstable_tasks == 1 - assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 - - def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): path = tmp_path / "multi.jsonl" outcomes = { diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 0b284d6c..66f18420 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -from pathlib import Path from typing import Any from evals import report as report_mod @@ -62,55 +61,132 @@ def test_print_table_shows_infra_errors(capsys): assert " 2" in out # i_err column value -def test_report_marks_entirely_estimated_result_token_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - } - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "estimated" - assert summary.tasks["R1"].result_tokens_mode == "estimated" - - report_mod.print_table(summary, "estimated") - output = capsys.readouterr().out - assert "entirely estimated" in output - assert "med_rtok~" in output - assert "~12" in output - - -def test_report_marks_mixed_measured_and_estimated_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], - "result_tokens_estimated": False, - }, - { - "task_id": "R1", - "rep": 1, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - }, - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "mixed" - assert summary.tasks["R1"].result_tokens_mode == "mixed" - - report_mod.print_table(summary, "mixed") - output = capsys.readouterr().out - assert "mixed measured and estimated" in output - assert "med_rtok*" in output +def test_report_behaviours(capsys, tmp_path): + def test_report_marks_entirely_estimated_result_token_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + } + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "estimated" + assert summary.tasks["R1"].result_tokens_mode == "estimated" + + report_mod.print_table(summary, "estimated") + output = capsys.readouterr().out + assert "entirely estimated" in output + assert "med_rtok~" in output + assert "~12" in output + + def test_report_marks_mixed_measured_and_estimated_columns(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], + "result_tokens_estimated": False, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + }, + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "mixed" + assert summary.tasks["R1"].result_tokens_mode == "mixed" + + report_mod.print_table(summary, "mixed") + output = capsys.readouterr().out + assert "mixed measured and estimated" in output + assert "med_rtok*" in output + + def test_report_main_table_cli(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + f2 = tmp_path / "b.jsonl" + f1.write_text( + json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", + encoding="utf-8", + ) + rc = report_mod.main(["--table", str(f1), str(f2)]) + assert rc == 0 + out = capsys.readouterr().out + assert "local" in out and "candidate" in out + assert "R1" in out + + def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path, capsys): + f1 = tmp_path / "old.jsonl" + f2 = tmp_path / "new.jsonl" + f1.write_text( + json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", + encoding="utf-8", + ) + + rc = report_mod.main(["--table", str(f1), str(f2)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "spans battery fingerprints" in captured.err + assert "different task prompts/questions" in captured.err + + def test_report_main_markdown_flag(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") + rc = report_mod.main(["--table", "--markdown", str(f1)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.startswith("| task |") + assert "| R1 |" in out + assert "---" in out + + def test_report_main_no_dedupe_flag(tmp_path, capsys): + p = tmp_path / "d.jsonl" + rows = [ + _synth_row("R1", label="local", num_calls=1, success=True), + {**_synth_row("R1", label="local", num_calls=9, success=False)}, + ] + # Both rows have the same (task_id, rep, label), so latest-wins keeps one. + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + rc = report_mod.main(["--no-dedupe", str(p)]) + assert rc == 0 + # With no-dedupe, both rows enter summarize → n=2 for R1. + # (dedupe default would leave n=1.) + out = capsys.readouterr().out + assert "R1" in out + assert "2/2" in out or "1/2" in out # one success of two + + test_report_marks_entirely_estimated_result_token_columns(capsys) + test_report_marks_mixed_measured_and_estimated_columns(capsys) + _d2 = tmp_path / "test_report_main_table_cli" + _d2.mkdir() + test_report_main_table_cli(_d2, capsys) + _d3 = tmp_path / "test_report_main_table_warns_when_battery_fingerprints_differ" + _d3.mkdir() + test_report_main_table_warns_when_battery_fingerprints_differ(_d3, capsys) + _d4 = tmp_path / "test_report_main_markdown_flag" + _d4.mkdir() + test_report_main_markdown_flag(_d4, capsys) + _d5 = tmp_path / "test_report_main_no_dedupe_flag" + _d5.mkdir() + test_report_main_no_dedupe_flag(_d5, capsys) def test_format_surface_cell_variants(): @@ -124,69 +200,72 @@ def test_format_surface_cell_variants(): assert format_surface_cell(_synth_row("R1", server="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" -def test_multi_surface_table_snapshot_with_external(): - local = [ - _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), - _synth_row("R2", label="local", success=False, num_calls=2), - ] - candidate = [ - _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), - _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), - ] - external = [ - _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), - _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), - _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), - ] - table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) - assert table["columns"] == ["local", "candidate", "akhil"] - assert "R1" in table["task_ids"] and "R3" in table["task_ids"] - assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" - assert table["cells"]["R1"]["candidate"] == "✅ 2c" - assert table["cells"]["R1"]["akhil"] == "✅ 3c" - assert table["cells"]["R2"]["candidate"] == "skip" - assert table["cells"]["R3"]["akhil"] == "ERR" - - text = render_multi_surface_table(table, markdown=False) - assert "local" in text and "candidate" in text and "akhil" in text - assert "✅ 3c" in text - assert "skip" in text - assert "ERR" in text - assert "infra 1" in text - - md = render_multi_surface_table(table, markdown=True) - assert md.startswith("| task |") - assert "| R1 |" in md - assert "---" in md - assert "**agg**" in md - - # Footer: external mispicks n/a - assert table["footer"]["akhil"]["mispicks"] is None - assert table["footer"]["local"]["mispicks"] == 1 - assert table["footer"]["akhil"]["infra_errors"] == 1 - - -def test_multi_surface_table_aggregates_reps_and_flags_unstable(): - rows = [ - _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), - _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), - _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), - _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), - _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), - _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), - ] - - table = build_multi_surface_table([("local", rows)]) - - assert table["multi_rep"] is True - assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" - assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" - assert table["footer"]["local"]["success"] == 5 - assert table["footer"]["local"]["n"] == 6 - assert table["footer"]["local"]["unstable_tasks"] == 1 - rendered = render_multi_surface_table(table) - assert "measured noise floor: 1 task flipped at least once" in rendered - assert "minimum meaningful difference: 2 tasks" in rendered +def test_multi_surface_behaviours(): + def test_multi_surface_table_snapshot_with_external(): + local = [ + _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), + _synth_row("R2", label="local", success=False, num_calls=2), + ] + candidate = [ + _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), + _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), + ] + external = [ + _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), + _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), + _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), + ] + table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) + assert table["columns"] == ["local", "candidate", "akhil"] + assert "R1" in table["task_ids"] and "R3" in table["task_ids"] + assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" + assert table["cells"]["R1"]["candidate"] == "✅ 2c" + assert table["cells"]["R1"]["akhil"] == "✅ 3c" + assert table["cells"]["R2"]["candidate"] == "skip" + assert table["cells"]["R3"]["akhil"] == "ERR" + + text = render_multi_surface_table(table, markdown=False) + assert "local" in text and "candidate" in text and "akhil" in text + assert "✅ 3c" in text + assert "skip" in text + assert "ERR" in text + assert "infra 1" in text + + md = render_multi_surface_table(table, markdown=True) + assert md.startswith("| task |") + assert "| R1 |" in md + assert "---" in md + assert "**agg**" in md + + # Footer: external mispicks n/a + assert table["footer"]["akhil"]["mispicks"] is None + assert table["footer"]["local"]["mispicks"] == 1 + assert table["footer"]["akhil"]["infra_errors"] == 1 + + def test_multi_surface_table_aggregates_reps_and_flags_unstable(): + rows = [ + _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), + _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), + _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), + _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), + _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), + _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), + ] + + table = build_multi_surface_table([("local", rows)]) + + assert table["multi_rep"] is True + assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" + assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" + assert table["footer"]["local"]["success"] == 5 + assert table["footer"]["local"]["n"] == 6 + assert table["footer"]["local"]["unstable_tasks"] == 1 + rendered = render_multi_surface_table(table) + assert "measured noise floor: 1 task flipped at least once" in rendered + assert "minimum meaningful difference: 2 tasks" in rendered + + test_multi_surface_table_snapshot_with_external() + test_multi_surface_table_aggregates_reps_and_flags_unstable() def test_single_rep_multi_surface_rendering_is_unchanged(): @@ -201,69 +280,3 @@ def test_single_rep_multi_surface_rendering_is_unchanged(): "-------------------------------------------------------\n" "local success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" ) - - -def test_report_main_table_cli(tmp_path: Path, capsys): - f1 = tmp_path / "a.jsonl" - f2 = tmp_path / "b.jsonl" - f1.write_text( - json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", - encoding="utf-8", - ) - f2.write_text( - json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", - encoding="utf-8", - ) - rc = report_mod.main(["--table", str(f1), str(f2)]) - assert rc == 0 - out = capsys.readouterr().out - assert "local" in out and "candidate" in out - assert "R1" in out - - -def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path: Path, capsys): - f1 = tmp_path / "old.jsonl" - f2 = tmp_path / "new.jsonl" - f1.write_text( - json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", - encoding="utf-8", - ) - f2.write_text( - json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", - encoding="utf-8", - ) - - rc = report_mod.main(["--table", str(f1), str(f2)]) - - assert rc == 0 - captured = capsys.readouterr() - assert "spans battery fingerprints" in captured.err - assert "different task prompts/questions" in captured.err - - -def test_report_main_markdown_flag(tmp_path: Path, capsys): - f1 = tmp_path / "a.jsonl" - f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") - rc = report_mod.main(["--table", "--markdown", str(f1)]) - assert rc == 0 - out = capsys.readouterr().out - assert out.startswith("| task |") - assert "| R1 |" in out - assert "---" in out - - -def test_report_main_no_dedupe_flag(tmp_path: Path, capsys): - p = tmp_path / "d.jsonl" - rows = [ - _synth_row("R1", label="local", num_calls=1, success=True), - {**_synth_row("R1", label="local", num_calls=9, success=False)}, - ] - # Both rows have the same (task_id, rep, label), so latest-wins keeps one. - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - rc = report_mod.main(["--no-dedupe", str(p)]) - assert rc == 0 - # With no-dedupe, both rows enter summarize → n=2 for R1. - # (dedupe default would leave n=1.) - out = capsys.readouterr().out - assert "R1" in out - assert "2/2" in out or "1/2" in out # one success of two diff --git a/tests/evals/runner/test_canary.py b/tests/evals/runner/test_canary.py index 111c3551..db98e493 100644 --- a/tests/evals/runner/test_canary.py +++ b/tests/evals/runner/test_canary.py @@ -5,6 +5,8 @@ import asyncio from unittest.mock import MagicMock +import pytest + from evals.runner import canary as runner_canary from evals.runner import ( run_canary, @@ -12,90 +14,96 @@ from evals.tasks.skip import TaskSkipped -def test_canary_detects_broken_verifier(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - - async def always_ok(plane, ctx, run): - return True, "false positive" +def test_canary_behaviours(monkeypatch): + def test_canary_detects_broken_verifier(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - async def correctly_fails(plane, ctx, run): - return False, "empty agent correctly rejected" + async def always_ok(plane, ctx, run): + return True, "false positive" - tasks = [ - { - "id": "GOOD", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": correctly_fails, - }, - { - "id": "BAD", - "prompt": "y {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": always_ok, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 + async def correctly_fails(plane, ctx, run): + return False, "empty agent correctly rejected" + tasks = [ + { + "id": "GOOD", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": correctly_fails, + }, + { + "id": "BAD", + "prompt": "y {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": always_ok, + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 1 -def test_canary_passes_when_all_verifiers_reject(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) + def test_canary_passes_when_all_verifiers_reject(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - async def reject(plane, ctx, run): - assert run == {"final_text": "", "calls": []} - return False, "no-op rejected" + async def reject(plane, ctx, run): + assert run == {"final_text": "", "calls": []} + return False, "no-op rejected" - tasks = [ - { - "id": "G1", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": reject, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 0 + tasks = [ + { + "id": "G1", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": reject, + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 0 + def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): + fake_plane = MagicMock() + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_canary, + "seed", + lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), + ) + monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) + tasks = [ + { + "id": "SKIPME", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (False, "unused"), + }, + ] + rc = asyncio.run(run_canary(tasks, label="local")) + assert rc == 1 -def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, - "seed", - lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - tasks = [ - { - "id": "SKIPME", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (False, "unused"), - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 + with pytest.MonkeyPatch.context() as mp: + test_canary_detects_broken_verifier(mp) + with pytest.MonkeyPatch.context() as mp: + test_canary_passes_when_all_verifiers_reject(mp) + with pytest.MonkeyPatch.context() as mp: + test_canary_exits_1_when_all_tasks_skipped(mp) diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 0862531e..8efe41ba 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -9,6 +9,7 @@ from typing import Any from unittest.mock import MagicMock +import pytest from plane.errors.errors import HttpError from evals import cli as run_mod @@ -46,24 +47,29 @@ def _taxonomy_task( } -def test_stdio_env_still_works_for_cli_drivers(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") - monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - env = stdio_server_env() - assert env["PLANE_API_KEY"] == "k" - assert "ANTHROPIC_API_KEY" not in env +def test_stdio_behaviours(monkeypatch): + def test_stdio_env_still_works_for_cli_drivers(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + env = stdio_server_env() + assert env["PLANE_API_KEY"] == "k" + assert "ANTHROPIC_API_KEY" not in env + def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): + monkeypatch.setenv("SOME_SECRET", "x") -def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): - monkeypatch.setenv("SOME_SECRET", "x") + environment = runner_live.stdio_server_env() - environment = runner_live.stdio_server_env() + assert "SOME_SECRET" not in environment + assert environment["PLANE_API_KEY"] == "test-key" + assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" + assert environment["PLANE_BASE_URL"] == "https://api.plane.so" - assert "SOME_SECRET" not in environment - assert environment["PLANE_API_KEY"] == "test-key" - assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" - assert environment["PLANE_BASE_URL"] == "https://api.plane.so" + with pytest.MonkeyPatch.context() as mp: + test_stdio_env_still_works_for_cli_drivers(mp) + with pytest.MonkeyPatch.context() as mp: + test_stdio_server_env_does_not_leak_ambient_secrets(mp) def test_live_run_rejects_non_positive_reps(capsys): @@ -71,573 +77,766 @@ def test_live_run_rejects_non_positive_reps(capsys): assert "--reps must be at least 1" in capsys.readouterr().err -def test_run_live_seed_failure_is_infra_seed(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" +def test_run_behaviours(tmp_path, monkeypatch, capsys): + def test_run_live_seed_failure_is_infra_seed(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - driver = MagicMock() - torn: list[dict[str, Any]] = [] - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + fake_plane = MagicMock() + driver = MagicMock() + torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - def boom_seed(plane, run_id, needs, ctx): - ctx["project_name"] = "EVAL deadbeef" - raise HttpError("identifier already taken", 409) + def boom_seed(plane, run_id, needs, ctx): + ctx["project_name"] = "EVAL deadbeef" + raise HttpError("identifier already taken", 409) - monkeypatch.setattr(runner_live, "seed", boom_seed) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + monkeypatch.setattr(runner_live, "seed", boom_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - task = { - "id": "T1", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - } + task = { + "id": "T1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + } - rc = asyncio.run( - run_live( - [task], - model_alias="standard", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - resolved_model_id="sonnet", + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + resolved_model_id="sonnet", + ) ) - ) - assert rc == 0 - rows = _data_rows(out) - assert len(rows) == 1 - row = rows[0] - assert row["schema_version"] == RESULT_SCHEMA_VERSION - assert row["error_class"] == "infra_seed" - assert row["success"] is False - assert row["verify_note"] == "" - assert "HttpError" in (row["error"] or "") - assert "identifier" in (row["error"] or "").lower() - assert row["battery"] # fingerprint written - assert row["requested_model"] == "standard" - assert row["requested_tier"] == "standard" - assert row["resolved_model"] == "sonnet" - assert row["model"] == "sonnet" - meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) - assert meta["schema_version"] == RESULT_SCHEMA_VERSION - assert meta["requested_tier"] == "standard" - assert meta["resolved_model"] == "sonnet" - driver.run_task.assert_not_called() - assert torn == [{"project_name": "EVAL deadbeef"}] - + assert rc == 0 + rows = _data_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["error_class"] == "infra_seed" + assert row["success"] is False + assert row["verify_note"] == "" + assert "HttpError" in (row["error"] or "") + assert "identifier" in (row["error"] or "").lower() + assert row["battery"] # fingerprint written + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "sonnet" + assert row["model"] == "sonnet" + meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert meta["schema_version"] == RESULT_SCHEMA_VERSION + assert meta["requested_tier"] == "standard" + assert meta["resolved_model"] == "sonnet" + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL deadbeef"}] + + def test_run_live_missing_bug_type_uses_context_skip_reason(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + def seed_without_bug_type(plane, run_id, needs, ctx): + ctx.update( + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "plan:work-item-types-disabled", + } + ) -def test_run_live_missing_bug_type_uses_context_skip_reason(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - def seed_without_bug_type(plane, run_id, needs, ctx): - ctx.update( + task = _taxonomy_task( + "BUGTYPE", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + needs={"bug_type"}, + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "plan:work-item-types-disabled" + assert row["verify_note"] == "plan:work-item-types-disabled" + assert row["error"] is None + assert row["error_class"] is None + driver.run_task.assert_not_called() + assert torn == [ { "project_name": "EVAL no bug type", "project_id": "p1", "bug_type_skip_reason": "plan:work-item-types-disabled", } + ] + + def test_run_live_prompt_bind_failure_is_infra_seed(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - task = _taxonomy_task( - "BUGTYPE", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - needs={"bug_type"}, - ) - rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + task = _taxonomy_task( + "PROMPT", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + prompt="use {missing_seed_id} in {project}", + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["verify_note"] == "" + assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] + + def test_run_live_api_driver_exception_is_infra_api(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.side_effect = RuntimeError("provider unavailable") + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - assert rc == 0 - row = _data_rows(out)[0] - assert row["skipped"] == "plan:work-item-types-disabled" - assert row["verify_note"] == "plan:work-item-types-disabled" - assert row["error"] is None - assert row["error_class"] is None - driver.run_task.assert_not_called() - assert torn == [ - { - "project_name": "EVAL no bug type", - "project_id": "p1", - "bug_type_skip_reason": "plan:work-item-types-disabled", - } - ] + task = _taxonomy_task( + "APIERR", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + ) + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="api", + resolved_model_id="provider-model-id", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_api" + assert row["verify_note"] == "" + assert row["success"] is False + assert row["error"] == "RuntimeError: provider unavailable" + driver.run_task.assert_called_once() + assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] -def test_run_live_prompt_bind_failure_is_infra_seed(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - torn: list[dict[str, Any]] = [] + def test_run_live_driver_exception_is_infra_cli(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + def ok_seed(plane, run_id, needs, ctx): + ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) - task = _taxonomy_task( - "PROMPT", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - prompt="use {missing_seed_id} in {project}", - ) - rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_seed" - assert row["verify_note"] == "" - assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") - driver.run_task.assert_not_called() - assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] + class BoomDriver: + name = "claude-cli" + def run_task(self, *args, **kwargs): + raise RuntimeError("claude cli failed: json_parse_failed") -def test_run_live_api_driver_exception_is_infra_api(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.side_effect = RuntimeError("provider unavailable") - torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + task = { + "id": "T2", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": lambda *a, **k: (False, "nope"), + } - task = _taxonomy_task( - "APIERR", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - ) - rc = asyncio.run( - run_live( - [task], - model_alias="standard", - reps=1, - label="local", - out_path=out, - driver_name="api", - resolved_model_id="provider-model-id", + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) ) - ) - - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_api" - assert row["verify_note"] == "" - assert row["success"] is False - assert row["error"] == "RuntimeError: provider unavailable" - driver.run_task.assert_called_once() - assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] - - -def test_run_live_driver_exception_is_infra_cli(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert "RuntimeError" in (row["error"] or "") + + def test_run_live_timeout_agent_is_infra_cli(tmp_path, monkeypatch): + from evals.results import agent_run_to_harness_dict + + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - def ok_seed(plane, run_id, needs, ctx): - ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) + class TimeoutDriver: + name = "claude-cli" - monkeypatch.setattr(runner_live, "seed", ok_seed) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="timeout", + notes=["timeout after 900s"], + ) - class BoomDriver: - name = "claude-cli" + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) - def run_task(self, *args, **kwargs): - raise RuntimeError("claude cli failed: json_parse_failed") + verify_calls: list[Any] = [] - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) + async def verify(*a, **k): + verify_calls.append(1) + return True, "should not run" - task = { - "id": "T2", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (False, "nope"), - } + task = { + "id": "T3", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert "RuntimeError" in (row["error"] or "") - - -def test_run_live_timeout_agent_is_infra_cli(tmp_path: Path, monkeypatch): - """Driver returns stopped_reason=timeout → row error_class=infra_cli, battery continues.""" - from evals.results import agent_run_to_harness_dict - - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - class TimeoutDriver: - name = "claude-cli" - - def run_task(self, *args, **kwargs): - return AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="timeout", - notes=["timeout after 900s"], + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed + assert row["stop_reason"] == "timeout" + assert verify_calls == [] + d = agent_run_to_harness_dict( + AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout"), + optimal=set(), + alternate=set(), + classify=lambda t, o, a: "out_of_set", + ) + assert d["stop_reason"] == "timeout" + + def test_run_live_error_during_execution_is_infra_cli(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "MCP server crashed", + "session_id": "sess-err", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") - verify_calls: list[Any] = [] + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) - async def verify(*a, **k): - verify_calls.append(1) - return True, "should not run" + verify_calls: list[Any] = [] - task = { - "id": "T3", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } + async def verify(*a, **k): + verify_calls.append(1) + return False, "nope" - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", + task = { + "id": "T4", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed - assert row["stop_reason"] == "timeout" - assert verify_calls == [] - d = agent_run_to_harness_dict( - AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout"), - optimal=set(), - alternate=set(), - classify=lambda t, o, a: "out_of_set", - ) - assert d["stop_reason"] == "timeout" - - -def test_run_live_error_during_execution_is_infra_cli(tmp_path: Path, monkeypatch): - """exit 1 + parseable JSON subtype error_during_execution → infra_cli; verify not called.""" - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - payload = { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": "MCP server crashed", - "session_id": "sess-err", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["stop_reason"] == "error_during_execution" + assert verify_calls == [] + assert "claude_exit=1" in (row.get("driver_notes") or []) + + def test_run_live_error_max_turns_is_task_path(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr( + runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + ) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_max_turns", + "is_error": True, + "result": "hit max turns", + "session_id": "sess-max", + "num_turns": 15, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) - verify_calls: list[Any] = [] + verify_calls: list[Any] = [] - async def verify(*a, **k): - verify_calls.append(1) - return False, "nope" + async def verify(*a, **k): + verify_calls.append(1) + return False, "agent exhausted turns" - task = { - "id": "T4", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", + task = { + "id": "T5", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert row["stop_reason"] == "error_during_execution" - assert verify_calls == [] - assert "claude_exit=1" in (row.get("driver_notes") or []) - - -def test_run_live_error_max_turns_is_task_path(tmp_path: Path, monkeypatch): - """exit 1 + subtype error_max_turns stays in the task denominator (not infra_cli).""" - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - payload = { - "type": "result", - "subtype": "error_max_turns", - "is_error": True, - "result": "hit max turns", - "session_id": "sess-max", - "num_turns": 15, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] is None + assert row["stop_reason"] == "error_max_turns" + assert row["success"] is False + assert verify_calls == [1] + + def test_run_live_verifier_skip_is_not_a_failure(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") + async def skip_verify(plane, ctx, run): + raise TaskSkipped("env:verification-unavailable") - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYSKIP", skip_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) - verify_calls: list[Any] = [] + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "env:verification-unavailable" + assert row["verify_note"] == "env:verification-unavailable" + assert row["success"] is False + assert row["error"] is None + assert row["error_class"] is None + assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] + + def test_run_live_verifier_exception_is_task_error(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] - async def verify(*a, **k): - verify_calls.append(1) - return False, "agent exhausted turns" + async def broken_verify(plane, ctx, run): + raise ValueError("verifier broke") - task = { - "id": "T5", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYERR", broken_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] is None - assert row["stop_reason"] == "error_max_turns" - assert row["success"] is False - assert verify_calls == [1] - -def test_run_live_verifier_skip_is_not_a_failure(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - torn: list[dict[str, Any]] = [] + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is False + assert row["error_class"] == "task" + assert row["error"] == "ValueError: verifier broke" + assert row["verify_note"] == "" + assert row["skipped"] is None + assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] + + def test_run_live_external_server_nulls_catalog_mispicks(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "search_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) - async def skip_verify(plane, ctx, run): - raise TaskSkipped("env:verification-unavailable") + async def verify_ok(plane, ctx, run): + return True, "external ok" - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("VERIFYSKIP", skip_verify)], - model_alias="standard", - reps=1, - label="local", - out_path=out, + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("EXTERNAL", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + server_cmd=["/bin/foreign", "stdio"], + ) ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["skipped"] == "env:verification-unavailable" - assert row["verify_note"] == "env:verification-unavailable" - assert row["success"] is False - assert row["error"] is None - assert row["error_class"] is None - assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["server"] == "external" + assert row["alternate_calls"] is None + assert row["out_of_set_calls"] is None + + def test_run_live_success_keeps_requested_and_resolved_models(tmp_path, monkeypatch): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "list_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + async def verify_ok(plane, ctx, run): + return True, "local ok" -def test_run_live_verifier_exception_is_task_error(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("SUCCESS", verify_ok)], + model_alias="standard", + resolved_model_id="provider-model-id", + reps=1, + label="local", + out_path=out, + ) + ) - async def broken_verify(plane, ctx, run): - raise ValueError("verifier broke") + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "provider-model-id" + assert row["server"] == "local" + + def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path, monkeypatch): + out = tmp_path / "multi.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_ids: list[str] = [] + teardown_projects: list[str] = [] + + def fresh_seed(plane, run_id, needs, ctx): + seed_ids.append(run_id) + ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + + def record_teardown(plane, ctx): + teardown_projects.append(ctx["project_id"]) + + async def fake_agent(**kwargs): + return TaskResult(final_text="done", stop_reason="end_turn") + + monkeypatch.setattr(runner_live, "seed", fresh_seed) + monkeypatch.setattr(runner_live, "teardown", record_teardown) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) + monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + task = { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "optimal_tools": {"list_work_items"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + } - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("VERIFYERR", broken_verify)], - model_alias="standard", - reps=1, - label="local", - out_path=out, + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=3, + label="local", + out_path=out, + driver_name="claude-cli", + ) ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is False - assert row["error_class"] == "task" - assert row["error"] == "ValueError: verifier broke" - assert row["verify_note"] == "" - assert row["skipped"] is None - assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] + assert rc == 0 + assert len(seed_ids) == 3 + assert len(set(seed_ids)) == 3 + assert teardown_projects == seed_ids + rows = _data_rows(out) + assert [row["rep"] for row in rows] == [0, 1, 2] + assert all(row["success"] is True for row in rows) + def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path): + from evals.runner import live as run_mod -def test_run_live_external_server_nulls_catalog_mispicks(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[{"tool": "search_work_items", "args": {}}], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) + captured: dict = {} - async def verify_ok(plane, ctx, run): - return True, "external ok" + def fake_get_driver(name, **kwargs): + captured["name"] = name + captured["kwargs"] = kwargs - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("EXTERNAL", verify_ok)], - model_alias="standard", - reps=1, - label="local", - out_path=out, - server_cmd=["/bin/foreign", "stdio"], - ) - ) + class Dummy: + def run_task(self, *a, **k): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + call_source="json", + ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is True - assert row["server"] == "external" - assert row["alternate_calls"] is None - assert row["out_of_set_calls"] is None + return Dummy() + monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) -def test_run_live_success_keeps_requested_and_resolved_models(tmp_path: Path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[{"tool": "list_work_items", "args": {}}], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) + import asyncio - async def verify_ok(plane, ctx, run): - return True, "local ok" + async def _verify(*a, **k): + return False, "n" - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("SUCCESS", verify_ok)], - model_alias="standard", - resolved_model_id="provider-model-id", - reps=1, - label="local", - out_path=out, + task = { + "id": "T", + "prompt": "x {project}", + "optimal_tools": {"a"}, + "alternate_tools": set(), + "optimal_calls": 1, + "needs": set(), + "verify": _verify, + } + rc = asyncio.run( + run_mod.run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=tmp_path / "o.jsonl", + driver_name="opencode-cli", + server_cmd=["/bin/foreign", "stdio"], + ) ) - ) - - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is True - assert row["requested_model"] == "standard" - assert row["requested_tier"] == "standard" - assert row["resolved_model"] == "provider-model-id" - assert row["server"] == "local" + assert rc == 0 + assert captured["name"] == "opencode-cli" + assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] + + def test_run_live_reports_progress_per_repetition(tmp_path, monkeypatch, capsys): + out = tmp_path / "out.jsonl" + + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + return TaskResult(final_text="done", num_calls=2) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + assert rc == 0 + + printed = capsys.readouterr().out + # Position out of total, before the task runs. + assert "[ 1/2] R1 rep=0 running" in printed + assert "[ 2/2] R2 rep=0 running" in printed + # A running tally after each, and one closing summary. + assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed + assert "finished 2/2 in " in printed + assert "2 pass, 0 fail, 0 skip" in printed + + _d0 = tmp_path / "test_run_live_seed_failure_is_infra_seed" + _d0.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_seed_failure_is_infra_seed(_d0, mp) + _d1 = tmp_path / "test_run_live_missing_bug_type_uses_context_skip_reason" + _d1.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_missing_bug_type_uses_context_skip_reason(_d1, mp) + _d2 = tmp_path / "test_run_live_prompt_bind_failure_is_infra_seed" + _d2.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_prompt_bind_failure_is_infra_seed(_d2, mp) + _d3 = tmp_path / "test_run_live_api_driver_exception_is_infra_api" + _d3.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_api_driver_exception_is_infra_api(_d3, mp) + _d4 = tmp_path / "test_run_live_driver_exception_is_infra_cli" + _d4.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_driver_exception_is_infra_cli(_d4, mp) + _d5 = tmp_path / "test_run_live_timeout_agent_is_infra_cli" + _d5.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_timeout_agent_is_infra_cli(_d5, mp) + _d6 = tmp_path / "test_run_live_error_during_execution_is_infra_cli" + _d6.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_error_during_execution_is_infra_cli(_d6, mp) + _d7 = tmp_path / "test_run_live_error_max_turns_is_task_path" + _d7.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_error_max_turns_is_task_path(_d7, mp) + _d8 = tmp_path / "test_run_live_verifier_skip_is_not_a_failure" + _d8.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_verifier_skip_is_not_a_failure(_d8, mp) + _d9 = tmp_path / "test_run_live_verifier_exception_is_task_error" + _d9.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_verifier_exception_is_task_error(_d9, mp) + _d10 = tmp_path / "test_run_live_external_server_nulls_catalog_mispicks" + _d10.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_external_server_nulls_catalog_mispicks(_d10, mp) + _d11 = tmp_path / "test_run_live_success_keeps_requested_and_resolved_models" + _d11.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_success_keeps_requested_and_resolved_models(_d11, mp) + _d12 = tmp_path / "test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep" + _d12.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(_d12, mp) + _d13 = tmp_path / "test_run_live_passes_server_cmd_to_non_claude" + _d13.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_passes_server_cmd_to_non_claude(mp, _d13) + _d14 = tmp_path / "test_run_live_reports_progress_per_repetition" + _d14.mkdir() + with pytest.MonkeyPatch.context() as mp: + test_run_live_reports_progress_per_repetition(_d14, mp, capsys) def test_is_infra_cli_stop_reason_matrix(): @@ -649,62 +848,6 @@ def test_is_infra_cli_stop_reason_matrix(): assert is_infra_cli_stop_reason("max_turns") is False -def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path: Path, monkeypatch): - out = tmp_path / "multi.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - seed_ids: list[str] = [] - teardown_projects: list[str] = [] - - def fresh_seed(plane, run_id, needs, ctx): - seed_ids.append(run_id) - ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) - - def record_teardown(plane, ctx): - teardown_projects.append(ctx["project_id"]) - - async def fake_agent(**kwargs): - return TaskResult(final_text="done", stop_reason="end_turn") - - monkeypatch.setattr(runner_live, "seed", fresh_seed) - monkeypatch.setattr(runner_live, "teardown", record_teardown) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) - monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) - - async def verify_ok(plane, ctx, run): - return True, "ok" - - task = { - "id": "R1", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify_ok, - } - - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=3, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - - assert rc == 0 - assert len(seed_ids) == 3 - assert len(set(seed_ids)) == 3 - assert teardown_projects == seed_ids - rows = _data_rows(out) - assert [row["rep"] for row in rows] == [0, 1, 2] - assert all(row["success"] is True for row in rows) - - def test_task_skipped_from_seed_records_a_skip_row(tmp_path: Path, monkeypatch): """A fixture that cannot be seeded records a skip — no agent, no crash. @@ -760,63 +903,6 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: assert summary.aggregate_n == 0 -def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path: Path): - """--server-cmd must not be Claude-only.""" - from evals.runner import live as run_mod - - captured: dict = {} - - def fake_get_driver(name, **kwargs): - captured["name"] = name - captured["kwargs"] = kwargs - - class Dummy: - def run_task(self, *a, **k): - return AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - call_source="json", - ) - - return Dummy() - - monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) - - import asyncio - - async def _verify(*a, **k): - return False, "n" - - task = { - "id": "T", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": _verify, - } - rc = asyncio.run( - run_mod.run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=tmp_path / "o.jsonl", - driver_name="opencode-cli", - server_cmd=["/bin/foreign", "stdio"], - ) - ) - assert rc == 0 - assert captured["name"] == "opencode-cli" - assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] - - # --------------------------------------------------------------------------- # Progress reporting # --------------------------------------------------------------------------- @@ -833,38 +919,3 @@ def test_elapsed_formats_minutes_then_hours(monkeypatch): assert _elapsed(1000.0) == "01:15" clock["now"] = 1000.0 + 3671 assert _elapsed(1000.0) == "1:01:11" - - -def test_run_live_reports_progress_per_repetition(tmp_path: Path, monkeypatch, capsys): - """A battery is tens of minutes long; each task must announce itself when it starts. - - Without this the operator sees nothing until the whole run ends, which is how - a stalled run looks identical to a working one. - """ - out = tmp_path / "out.jsonl" - - async def passes(_plane, _ctx, _run): - return True, "ok" - - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - async def fake_drive(**kwargs): - return TaskResult(final_text="done", num_calls=2) - - monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) - monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) - - tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] - rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) - assert rc == 0 - - printed = capsys.readouterr().out - # Position out of total, before the task runs. - assert "[ 1/2] R1 rep=0 running" in printed - assert "[ 2/2] R2 rep=0 running" in printed - # A running tally after each, and one closing summary. - assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed - assert "finished 2/2 in " in printed - assert "2 pass, 0 fail, 0 skip" in printed diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py index 1c60c472..0493a3eb 100644 --- a/tests/evals/runner/test_resume.py +++ b/tests/evals/runner/test_resume.py @@ -25,136 +25,198 @@ from tests.evals.conftest import _data_rows -def test_should_skip_resume_row_completed_success(): - assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True - - -def test_should_skip_resume_row_verify_fail_without_error(): - # Completed attempt (agent ran, verify failed) — do not re-run on resume. - assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True - - -def test_should_skip_resume_row_infra_seed_retries(): - assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False - - -def test_should_skip_resume_row_infra_cli_retries(): - assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False - - -def test_should_skip_resume_row_non_null_error_retries(): - assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False - assert should_skip_resume_row({"error": "boom", "error_class": None}) is False - - -def test_load_resume_skip_keys_summary(tmp_path: Path): - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, - {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, - {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local"), ("W1", 0, "local")} - assert n_skip == 2 - assert n_retry == 1 - - -def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path: Path): - """Historical error row whose later row succeeded must not inflate n_retry.""" - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - assert n_retry == 0 - - -def test_load_resume_skip_keys_label_mismatch(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") - with pytest.raises(SystemExit, match="label"): - load_resume_skip_keys(p, label="local") - - -def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "battery": "aaaaaaaaaaaa", - "model": "sonnet", - "driver": "claude-cli", - "error": None, - } +def test_should_skip_behaviours(): + def test_should_skip_resume_row_completed_success(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True + + def test_should_skip_resume_row_verify_fail_without_error(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True + + def test_should_skip_resume_row_infra_seed_retries(): + assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False + + def test_should_skip_resume_row_infra_cli_retries(): + assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False + + def test_should_skip_resume_row_non_null_error_retries(): + assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False + assert should_skip_resume_row({"error": "boom", "error_class": None}) is False + + test_should_skip_resume_row_completed_success() + test_should_skip_resume_row_verify_fail_without_error() + test_should_skip_resume_row_infra_seed_retries() + test_should_skip_resume_row_infra_cli_retries() + test_should_skip_resume_row_non_null_error_retries() + + +def test_load_behaviours(tmp_path, capsys): + def test_load_resume_skip_keys_summary(tmp_path): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local"), ("W1", 0, "local")} + assert n_skip == 2 + assert n_retry == 1 + + def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + assert n_retry == 0 + + def test_load_resume_skip_keys_label_mismatch(tmp_path): + p = tmp_path / "out.jsonl" + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") + with pytest.raises(SystemExit, match="label"): + load_resume_skip_keys(p, label="local") + + def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "aaaaaaaaaaaa", + "model": "sonnet", + "driver": "claude-cli", + "error": None, + } + ) + + "\n", + encoding="utf-8", ) - + "\n", - encoding="utf-8", - ) - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") - with pytest.raises(SystemExit, match="driver"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") - # Missing keys on older rows: pass (back-compat) - p2 = tmp_path / "old.jsonl" - p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") - skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") - assert ("R1", 0, "local") in skip - - -def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path: Path): - p = tmp_path / "tiered.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "model": "provider-reported-id", - "requested_model": "standard", - "requested_tier": "standard", - "resolved_model": "old-standard-id", - "error": None, - } + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") + with pytest.raises(SystemExit, match="driver"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") + # Missing keys on older rows: pass (back-compat) + p2 = tmp_path / "old.jsonl" + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0, "local") in skip + + def test_load_resume_skip_keys_truncated_json(tmp_path, capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + + "\n" + + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + err = capsys.readouterr().err + assert "invalid JSON" in err + + def test_load_resume_skip_keys_missing_file(tmp_path): + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") + assert skip == set() and n_skip == 0 and n_retry == 0 + + _d0 = tmp_path / "test_load_resume_skip_keys_summary" + _d0.mkdir() + test_load_resume_skip_keys_summary(_d0) + _d1 = tmp_path / "test_load_resume_skip_keys_n_retry_ignores_later_success" + _d1.mkdir() + test_load_resume_skip_keys_n_retry_ignores_later_success(_d1) + _d2 = tmp_path / "test_load_resume_skip_keys_label_mismatch" + _d2.mkdir() + test_load_resume_skip_keys_label_mismatch(_d2) + _d3 = tmp_path / "test_load_resume_skip_keys_battery_model_driver_mismatch" + _d3.mkdir() + test_load_resume_skip_keys_battery_model_driver_mismatch(_d3) + _d4 = tmp_path / "test_load_resume_skip_keys_truncated_json" + _d4.mkdir() + test_load_resume_skip_keys_truncated_json(_d4, capsys) + _d5 = tmp_path / "test_load_resume_skip_keys_missing_file" + _d5.mkdir() + test_load_resume_skip_keys_missing_file(_d5) + + +def test_resume_behaviours(tmp_path): + def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path): + p = tmp_path / "tiered.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "model": "provider-reported-id", + "requested_model": "standard", + "requested_tier": "standard", + "resolved_model": "old-standard-id", + "error": None, + } + ) + + "\n", + encoding="utf-8", ) - + "\n", - encoding="utf-8", - ) - - skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") - assert skip == {("R1", 0, "local")} - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", model="new-standard-id") - -def test_load_resume_skip_keys_truncated_json(tmp_path: Path, capsys): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) - + "\n" - + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated - encoding="utf-8", - ) - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - err = capsys.readouterr().err - assert "invalid JSON" in err + skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") + assert skip == {("R1", 0, "local")} + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", model="new-standard-id") + + def test_resume_skips_meta_and_mismatch_checks_it(tmp_path): + p = tmp_path / "out.jsonl" + p.write_text( + "\n".join( + [ + json.dumps( + { + "row_type": "meta", + "label": "candidate", + "battery": "bbbbbbbbbbbb", + "model": "sonnet", + "driver": "claude-cli", + } + ), + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "candidate", + "error": None, + "error_class": None, + "success": True, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys( + p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + ) + assert skip == {("R1", 0, "candidate")} + assert n_skip == 1 and n_retry == 0 + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") -def test_load_resume_skip_keys_missing_file(tmp_path: Path): - skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") - assert skip == set() and n_skip == 0 and n_retry == 0 + _d0 = tmp_path / "test_resume_identity_uses_resolved_model_not_tier_label" + _d0.mkdir() + test_resume_identity_uses_resolved_model_not_tier_label(_d0) + _d1 = tmp_path / "test_resume_skips_meta_and_mismatch_checks_it" + _d1.mkdir() + test_resume_skips_meta_and_mismatch_checks_it(_d1) def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): @@ -285,42 +347,3 @@ def test_make_run_meta_row_and_write_once(tmp_path: Path): assert len(lines) == 2 assert json.loads(lines[0])["row_type"] == "meta" assert json.loads(lines[1])["task_id"] == "R1" - - -def test_resume_skips_meta_and_mismatch_checks_it(tmp_path: Path): - p = tmp_path / "out.jsonl" - p.write_text( - "\n".join( - [ - json.dumps( - { - "row_type": "meta", - "label": "candidate", - "battery": "bbbbbbbbbbbb", - "model": "sonnet", - "driver": "claude-cli", - } - ), - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "candidate", - "error": None, - "error_class": None, - "success": True, - } - ), - ] - ) - + "\n", - encoding="utf-8", - ) - skip, n_skip, n_retry = load_resume_skip_keys( - p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" - ) - assert skip == {("R1", 0, "candidate")} - assert n_skip == 1 and n_retry == 0 - - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") diff --git a/tests/evals/seed/test_plan_gate.py b/tests/evals/seed/test_plan_gate.py index fa4896a3..63381338 100644 --- a/tests/evals/seed/test_plan_gate.py +++ b/tests/evals/seed/test_plan_gate.py @@ -1,14 +1,11 @@ -"""Characterization of `is_plan_gate` against the refusals Plane actually returns. +"""Characterization of `is_plan_gate` against the payloads `api/` actually returns. -Every payload below is the real shape from `plane-ee/apps/api/plane/api/` — the v1 -external layer the SDK talks to — rather than an invented one. A plan gate makes the -harness record an environment skip; anything else must stay a real error, so a -misclassification here either hides a defect or reports one that is not there. +A gate becomes an environment skip and anything else stays a real error, so a +misclassification here either hides a defect or invents one. """ from __future__ import annotations -import pytest from plane.errors.errors import HttpError from evals.seed import is_plan_gate @@ -16,111 +13,78 @@ # --- genuine plan refusals ------------------------------------------------------------ PLAN_GATES = [ - pytest.param( + ( HttpError("Payment required", 402, {"error": "Payment required", "error_code": 1999}), - id="402-check_feature_flag-decorator", + "402-check_feature_flag-decorator", ), - pytest.param( - HttpError("Payment required", 402, None), - id="402-with-no-body", - ), - pytest.param( + (HttpError("Payment required", 402, None), "402-with-no-body"), + ( HttpError( "Forbidden", 403, {"detail": "Payment required. Upgrade your plan to access Initiatives"}, ), - id="403-initiatives-permission-class", + "403-initiatives-permission-class", ), - pytest.param( + ( HttpError( "Forbidden", 403, {"detail": "Payment required. Upgrade your plan to access Teamspaces"}, ), - id="403-teamspaces-permission-class", - ), - pytest.param( - HttpError("Bad request", 400, {"error": "Upgrade your plan to enable formula properties"}), - id="400-with-plan-prose", + "403-teamspaces-permission-class", ), + (HttpError("Bad request", 400, {"error": "Upgrade your plan to enable formula properties"}), "400-with-plan-prose"), ] # --- refusals that are NOT plan limits ------------------------------------------------- NOT_PLAN_GATES = [ - pytest.param( + ( HttpError("Forbidden", 403, {"detail": "You don't have permission to create this project"}), - id="bare-403-is-rbac-not-a-plan-limit", + "bare-403-is-rbac-not-a-plan-limit", ), - pytest.param( + ( HttpError( "Forbidden", 403, {"error": "Customer feature is not enabled for this workspace"}, ), - id="403-customer-toggle-is-configuration-the-harness-controls", - ), - pytest.param( - HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}), - id="404-worklog-toggle", + "403-customer-toggle-is-configuration-the-harness-controls", ), - pytest.param( + (HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}), "404-worklog-toggle"), + ( HttpError("Bad request", 400, {"non_field_errors": ["Cycles are not enabled for this project"]}), - id="400-cycle-toggle", + "400-cycle-toggle", ), - pytest.param( + ( HttpError("Bad request", 400, {"non_field_errors": ["Modules are not enabled for this project"]}), - id="400-module-toggle", - ), - pytest.param( - HttpError("Bad request", 400, {"name": ["This field is required."]}), - id="400-ordinary-validation-error", - ), - pytest.param( - HttpError("Server error", 500, {"error": "Internal server error"}), - id="500-never-a-gate", - ), - pytest.param( - HttpError("Too many requests", 429, {"error": "Rate limit exceeded"}), - id="429-never-a-gate", + "400-module-toggle", ), + (HttpError("Bad request", 400, {"name": ["This field is required."]}), "400-ordinary-validation-error"), + (HttpError("Server error", 500, {"error": "Internal server error"}), "500-never-a-gate"), + (HttpError("Too many requests", 429, {"error": "Rate limit exceeded"}), "429-never-a-gate"), ] -@pytest.mark.parametrize("exc", PLAN_GATES) -def test_plan_refusals_are_gates(exc): - assert is_plan_gate(exc) is True - - -@pytest.mark.parametrize("exc", NOT_PLAN_GATES) -def test_other_refusals_are_not_gates(exc): - assert is_plan_gate(exc) is False +def test_only_refusals_that_name_a_plan_limit_are_gates(): + """402 is unambiguous; 403/400 need the body to say so, since 403 is also plain RBAC.""" + for exc, label in PLAN_GATES: + assert is_plan_gate(exc) is True, label + for exc, label in NOT_PLAN_GATES: + assert is_plan_gate(exc) is False, label -@pytest.mark.parametrize( - "exc", - [ - pytest.param(RuntimeError("connection reset"), id="non-http-exception"), - pytest.param(TimeoutError(), id="timeout"), - pytest.param(ValueError("upgrade your plan"), id="non-http-even-with-plan-wording"), - ], -) -def test_non_http_exceptions_are_never_gates(exc): - """A transport failure must surface as infrastructure, not be excused as a plan limit.""" - assert is_plan_gate(exc) is False +def test_non_http_exceptions_are_never_gates(): + """A transport failure is infrastructure, not a plan limit to be excused.""" + for exc in (RuntimeError("connection reset"), TimeoutError(), ValueError("upgrade your plan")): + assert is_plan_gate(exc) is False, repr(exc) def test_a_bare_403_would_previously_have_been_swallowed(): - """The regression this tightening exists for. - - An RBAC denial and an initiatives plan gate are both 403 with a ``detail`` string. - Classifying on status alone turned a permission bug into an environment skip, which - reads as 'nothing to see here' in every report that excludes skips from denominators. - """ + """The regression this exists for: RBAC denial and a plan gate share status and shape.""" rbac = HttpError("Forbidden", 403, {"detail": "You don't have permission to view this issue"}) gate = HttpError("Forbidden", 403, {"detail": "Payment required. Upgrade your plan to access Initiatives"}) - assert rbac.status_code == gate.status_code assert is_plan_gate(rbac) is False assert is_plan_gate(gate) is True diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index 5c6ffef9..1aa60315 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -61,161 +61,358 @@ def __init__(self): self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) -def test_seed_plan_covers_all_groups(): - groups = { - "items", - "labels", - "bug_type", - "cycles", - "module", - "intake", - "customer", - "release", - "second_project", - } - lines = seed_plan(groups) - blob = "\n".join(lines) - for g in groups: - assert ( - g.split("_")[0] in blob or g in blob or g.replace("_", " ") in blob or any(g in line for line in lines) - ), f"seed_plan missing {g}: {lines}" - # Specific fixtures named - assert "Sprint 12" in blob - assert "Checkout revamp" in blob - assert "1.2.0" in blob - assert "Acme Corp" in blob - - -def test_seed_plan_empty_needs_only_project(): - lines = seed_plan(set()) - assert any("project" in line for line in lines) - # project line + default workspace customers enable note - assert any("customers" in line for line in lines) - assert len(lines) == 2 - - -def test_seed_module_ast_has_all_group_handlers(): - """seed() dispatches every documented fixture group.""" - src = inspect.getsource(seed_mod.seed) - for group in ( - "labels", - "items", - "bug_type", - "cycles", - "module", - "intake", - "customer", - "release", - "second_project", - ): - assert f'"{group}"' in src or f"'{group}'" in src, group - - -def test_seed_enables_project_features_immediately_after_create(monkeypatch): - """Fresh projects ship with cycles/modules/intake/worklogs off — seed must enable them. - - Sequence: create → workspace features (customers) → project update → project features. - """ - from types import SimpleNamespace - - from plane.models.projects import ProjectFeature, UpdateProject - from plane.models.workspaces import WorkspaceFeature - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - calls: list[tuple] = [] - - class _Projects: - def create(self, workspace_slug, data): - calls.append(("create", workspace_slug, getattr(data, "name", None))) - return SimpleNamespace(id="proj-main") - - def update(self, workspace_slug, project_id, data): - assert isinstance(data, UpdateProject) - calls.append(("update", project_id, data.model_dump(exclude_none=True))) - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - assert isinstance(data, ProjectFeature) - calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) - return data +def test_seed_behaviours(monkeypatch): + def test_seed_plan_behaviours(): + def test_seed_plan_covers_all_groups(): + groups = { + "items", + "labels", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + } + lines = seed_plan(groups) + blob = "\n".join(lines) + for g in groups: + assert ( + g.split("_")[0] in blob + or g in blob + or g.replace("_", " ") in blob + or any(g in line for line in lines) + ), f"seed_plan missing {g}: {lines}" + # Specific fixtures named + assert "Sprint 12" in blob + assert "Checkout revamp" in blob + assert "1.2.0" in blob + assert "Acme Corp" in blob + + def test_seed_plan_empty_needs_only_project(): + lines = seed_plan(set()) + assert any("project" in line for line in lines) + # project line + default workspace customers enable note + assert any("customers" in line for line in lines) + assert len(lines) == 2 + + test_seed_plan_covers_all_groups() + test_seed_plan_empty_needs_only_project() + + def test_seed_module_ast_has_all_group_handlers(): + src = inspect.getsource(seed_mod.seed) + for group in ( + "labels", + "items", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + ): + assert f'"{group}"' in src or f"'{group}'" in src, group + + def test_seed_enables_project_features_immediately_after_create(monkeypatch): + from types import SimpleNamespace + + from plane.models.projects import ProjectFeature, UpdateProject + from plane.models.workspaces import WorkspaceFeature + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + calls.append(("create", workspace_slug, getattr(data, "name", None))) + return SimpleNamespace(id="proj-main") + + def update(self, workspace_slug, project_id, data): + assert isinstance(data, UpdateProject) + calls.append(("update", project_id, data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + assert isinstance(data, ProjectFeature) + calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(("ws_update_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx) + + assert ctx["project_id"] == "proj-main" + kinds = [c[0] for c in calls] + assert kinds == ["create", "ws_update_features", "update", "update_features"] + # Workspace customers enabled for C1 preconditions + assert calls[1][1].get("customers") is True + assert "work_item_types" not in calls[1][1] + # Project enable calls target the created id + assert calls[2][1] == "proj-main" + assert calls[3][1] == "proj-main" + upd = calls[2][2] + assert upd.get("cycle_view") is True + assert upd.get("is_time_tracking_enabled") is True + feat = calls[3][2] + assert feat.get("cycles") is True + + def test_seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-s5") + + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": True}) + + def update_features(self, workspace_slug, data): + calls.append(("ws_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) + assert ctx["feature_exclude"] == ["cycles", "worklogs"] + assert ctx["ws_feature_exclude"] == ["customers"] + assert ctx["s5_left_customers_off"] is True + # Excluded features are written OFF, not omitted. The workspace outlives the run, so + # omitting the write leaves the previous rep's value and S5's precondition never holds. + ws = next(c[1] for c in calls if c[0] == "ws_features") + assert ws.get("customers") is False + assert ctx["workspace_features_prior"] == {"customers": True} + upd = next(c[1] for c in calls if c[0] == "update") + assert upd.get("cycle_view") is False + assert upd.get("is_time_tracking_enabled") is False + assert upd.get("module_view") is True + feat = next(c[1] for c in calls if c[0] == "features") + assert feat.get("cycles") is False + assert feat.get("modules") is True + + def test_seed_cycles_create_add_then_backdate(monkeypatch): + from types import SimpleNamespace + + from plane.models.cycles import CreateCycle, UpdateCycle + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + cycle_seq = {"n": 0} + + class _Cycles: + def create(self, workspace_slug, project_id, data): + assert isinstance(data, CreateCycle) + cycle_seq["n"] += 1 + cid = f"cyc-{cycle_seq['n']}" + calls.append( + ( + "create", + { + "name": data.name, + "start_date": data.start_date, + "end_date": data.end_date, + "id": cid, + }, + ) + ) + return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) - class _Workspaces: - def update_features(self, workspace_slug, data): - assert isinstance(data, WorkspaceFeature) - calls.append(("ws_update_features", data.model_dump(exclude_none=True))) - return data + def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): + calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) - plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) - ctx: dict = {} - seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx) - - assert ctx["project_id"] == "proj-main" - kinds = [c[0] for c in calls] - assert kinds == ["create", "ws_update_features", "update", "update_features"] - # Workspace customers enabled for C1 preconditions - assert calls[1][1].get("customers") is True - assert "work_item_types" not in calls[1][1] - # Project enable calls target the created id - assert calls[2][1] == "proj-main" - assert calls[3][1] == "proj-main" - upd = calls[2][2] - assert upd.get("cycle_view") is True - assert upd.get("is_time_tracking_enabled") is True - feat = calls[3][2] - assert feat.get("cycles") is True + def update(self, workspace_slug, project_id, cycle_id, data): + assert isinstance(data, UpdateCycle) + calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) + return SimpleNamespace(id=cycle_id, end_date=data.end_date) + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-1") -def test_seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): - """S5 needs leave_cycles_worklogs_off — project cycles/worklogs + workspace customers OFF.""" - from types import SimpleNamespace + def update(self, workspace_slug, project_id, data): + return SimpleNamespace(id=project_id) - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) + def update_features(self, workspace_slug, project_id, data): + return data - calls: list[tuple] = [] + class _Workspaces: + def update_features(self, workspace_slug, data): + return data - class _Projects: - def create(self, workspace_slug, data): - return SimpleNamespace(id="proj-s5") + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {}) - def update(self, workspace_slug, project_id, data): - calls.append(("update", data.model_dump(exclude_none=True))) - return SimpleNamespace(id=project_id) + class _Users: + def get_me(self): + return SimpleNamespace(id="user-1") - def update_features(self, workspace_slug, project_id, data): - calls.append(("features", data.model_dump(exclude_none=True))) - return data + class _States: + def list(self, workspace_slug, project_id): + return SimpleNamespace( + results=[ + SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), + ] + ) - class _Workspaces: - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {"customers": True}) + item_n = {"n": 0} - def update_features(self, workspace_slug, data): - calls.append(("ws_features", data.model_dump(exclude_none=True))) - return data + class _WorkItems: + def create(self, workspace_slug, project_id, data): + item_n["n"] += 1 + return SimpleNamespace( + id=f"wi-{item_n['n']}", + name=data.name, + state="st-started", + created_at="2026-01-01", + ) - plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) - ctx: dict = {} - seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) - assert ctx["feature_exclude"] == ["cycles", "worklogs"] - assert ctx["ws_feature_exclude"] == ["customers"] - assert ctx["s5_left_customers_off"] is True - # Excluded features are written OFF, not omitted. The workspace outlives the run, so - # omitting the write leaves the previous rep's value and S5's precondition never holds. - ws = next(c[1] for c in calls if c[0] == "ws_features") - assert ws.get("customers") is False - assert ctx["workspace_features_prior"] == {"customers": True} - upd = next(c[1] for c in calls if c[0] == "update") - assert upd.get("cycle_view") is False - assert upd.get("is_time_tracking_enabled") is False - assert upd.get("module_view") is True - feat = next(c[1] for c in calls if c[0] == "features") - assert feat.get("cycles") is False - assert feat.get("modules") is True + def update(self, workspace_slug, project_id, work_item_id, data): + return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) + + class comments: + @staticmethod + def create(**kw): + return SimpleNamespace(id="c1") + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + cycles=_Cycles(), + users=_Users(), + states=_States(), + work_items=_WorkItems(), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) + + # Filter to Sprint-12-related create/add/update sequence (first cycle is past). + past_id = ctx["cycle_past_id"] + # Must create both cycles before any backdate update of past. + create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] + assert len(create_idxs) == 2 + past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) + # Created with temporary *future* end_date (active), not the final past end. + assert past_create[1]["end_date"] > past_create[1]["start_date"] + # At least one add to past cycle before its update + past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] + past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] + assert past_adds, "expected add_work_items on Sprint 12" + assert past_updates, "expected backdate update on Sprint 12" + assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" + # Backdated end matches W6 seed ctx; differs from create-time active end + backdated_end = calls[past_updates[0]][1]["end_date"] + assert ctx["cycle_past_seed_end_date"] == backdated_end + assert backdated_end != past_create[1]["end_date"] + assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] + # Active cycle: create with future end; never backdated + cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) + assert cur_create[1]["end_date"] + cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] + assert cur_updates == [] + + def test_seed_enables_features_on_second_project_too(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + creates: list[str] = [] + enables: list[str] = [] + + class _Projects: + def create(self, workspace_slug, data): + pid = f"p-{len(creates)}" + creates.append(pid) + return SimpleNamespace(id=pid) + + def update(self, workspace_slug, project_id, data): + enables.append(("update", project_id)) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + enables.append(("features", project_id)) + return data + + # Minimal stubs so second_project seed gets past bug_type + work items. + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) + + def update_features(self, workspace_slug, data): + enables.append(("ws_features", workspace_slug)) + return data + + class _WorkItemTypes: + def list(self, **kw): + return [SimpleNamespace(id="bug-1", name="Bug")] + + def create(self, **kw): + return SimpleNamespace(id="bug-1", name="Bug") + + def import_to_project(self, **kw): + return None + + class _WorkItems: + def create(self, **kw): + return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + work_item_types=_WorkItemTypes(), + work_items=_WorkItems(), + ) + ctx: dict = {} + # second_project path also seeds bug_type when missing + seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) + + assert len(creates) == 2 + # Each create followed by update + update_features for that project id + assert ("update", creates[0]) in enables + assert ("features", creates[0]) in enables + assert ("update", creates[1]) in enables + assert ("features", creates[1]) in enables + + test_seed_plan_behaviours() + test_seed_module_ast_has_all_group_handlers() + with pytest.MonkeyPatch.context() as mp: + test_seed_enables_project_features_immediately_after_create(mp) + with pytest.MonkeyPatch.context() as mp: + test_seed_s5_leaves_cycles_worklogs_and_customers_off(mp) + with pytest.MonkeyPatch.context() as mp: + test_seed_cycles_create_add_then_backdate(mp) + with pytest.MonkeyPatch.context() as mp: + test_seed_enables_features_on_second_project_too(mp) def test_excluding_pages_turns_page_view_off_despite_its_true_default(monkeypatch): @@ -251,135 +448,6 @@ def update_features(self, workspace_slug, project_id, data): assert feat.get("cycles") is True -def test_seed_cycles_create_add_then_backdate(monkeypatch): - """Sprint 12: create (active end) → add_work_items → update(end_date past). - - Plane rejects adds when end_date is already past; seed must not create past first. - """ - from types import SimpleNamespace - - from plane.models.cycles import CreateCycle, UpdateCycle - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - calls: list[tuple] = [] - cycle_seq = {"n": 0} - - class _Cycles: - def create(self, workspace_slug, project_id, data): - assert isinstance(data, CreateCycle) - cycle_seq["n"] += 1 - cid = f"cyc-{cycle_seq['n']}" - calls.append( - ( - "create", - { - "name": data.name, - "start_date": data.start_date, - "end_date": data.end_date, - "id": cid, - }, - ) - ) - return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) - - def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): - calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) - - def update(self, workspace_slug, project_id, cycle_id, data): - assert isinstance(data, UpdateCycle) - calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) - return SimpleNamespace(id=cycle_id, end_date=data.end_date) - - class _Projects: - def create(self, workspace_slug, data): - return SimpleNamespace(id="proj-1") - - def update(self, workspace_slug, project_id, data): - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - return data - - class _Workspaces: - def update_features(self, workspace_slug, data): - return data - - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {}) - - class _Users: - def get_me(self): - return SimpleNamespace(id="user-1") - - class _States: - def list(self, workspace_slug, project_id): - return SimpleNamespace( - results=[ - SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), - SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), - ] - ) - - item_n = {"n": 0} - - class _WorkItems: - def create(self, workspace_slug, project_id, data): - item_n["n"] += 1 - return SimpleNamespace( - id=f"wi-{item_n['n']}", - name=data.name, - state="st-started", - created_at="2026-01-01", - ) - - def update(self, workspace_slug, project_id, work_item_id, data): - return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) - - class comments: - @staticmethod - def create(**kw): - return SimpleNamespace(id="c1") - - plane = SimpleNamespace( - projects=_Projects(), - workspaces=_Workspaces(), - cycles=_Cycles(), - users=_Users(), - states=_States(), - work_items=_WorkItems(), - ) - ctx: dict = {} - seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) - - # Filter to Sprint-12-related create/add/update sequence (first cycle is past). - past_id = ctx["cycle_past_id"] - # Must create both cycles before any backdate update of past. - create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] - assert len(create_idxs) == 2 - past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) - # Created with temporary *future* end_date (active), not the final past end. - assert past_create[1]["end_date"] > past_create[1]["start_date"] - # At least one add to past cycle before its update - past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] - past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] - assert past_adds, "expected add_work_items on Sprint 12" - assert past_updates, "expected backdate update on Sprint 12" - assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" - # Backdated end matches W6 seed ctx; differs from create-time active end - backdated_end = calls[past_updates[0]][1]["end_date"] - assert ctx["cycle_past_seed_end_date"] == backdated_end - assert backdated_end != past_create[1]["end_date"] - assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] - # Active cycle: create with future end; never backdated - cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) - assert cur_create[1]["end_date"] - cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] - assert cur_updates == [] - - @pytest.mark.parametrize("prior", [True, False]) def test_teardown_restores_the_workspace_value_it_found(monkeypatch, prior): """Teardown puts the toggle back, rather than forcing the value this run wanted. @@ -415,281 +483,228 @@ def update_features(self, workspace_slug, data): assert calls and calls[0].get("customers") is prior -def test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown(): - """An unreadable prior value must not become a guess written back to the workspace.""" - from types import SimpleNamespace - - calls: list = [] - - class _Workspaces: - def update_features(self, workspace_slug, data): - calls.append(data) - return data - - plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) - seed_mod.teardown( - plane, - { - "workspace_slug": "test-ws", - "workspace_features_prior": {"customers": None}, - "project_id": None, - }, - ) - assert calls == [] - - -def test_seed_enables_features_on_second_project_too(monkeypatch): - """R6 second project also gets feature enable after its create.""" - from types import SimpleNamespace - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - creates: list[str] = [] - enables: list[str] = [] +def test_teardown_behaviours(): + def test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown(): + from types import SimpleNamespace - class _Projects: - def create(self, workspace_slug, data): - pid = f"p-{len(creates)}" - creates.append(pid) - return SimpleNamespace(id=pid) + calls: list = [] - def update(self, workspace_slug, project_id, data): - enables.append(("update", project_id)) - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - enables.append(("features", project_id)) - return data + class _Workspaces: + def update_features(self, workspace_slug, data): + calls.append(data) + return data - # Minimal stubs so second_project seed gets past bug_type + work items. - class _Workspaces: - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) - - def update_features(self, workspace_slug, data): - enables.append(("ws_features", workspace_slug)) - return data - - class _WorkItemTypes: - def list(self, **kw): - return [SimpleNamespace(id="bug-1", name="Bug")] - - def create(self, **kw): - return SimpleNamespace(id="bug-1", name="Bug") - - def import_to_project(self, **kw): - return None - - class _WorkItems: - def create(self, **kw): - return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) - - plane = SimpleNamespace( - projects=_Projects(), - workspaces=_Workspaces(), - work_item_types=_WorkItemTypes(), - work_items=_WorkItems(), - ) - ctx: dict = {} - # second_project path also seeds bug_type when missing - seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) - - assert len(creates) == 2 - # Each create followed by update + update_features for that project id - assert ("update", creates[0]) in enables - assert ("features", creates[0]) in enables - assert ("update", creates[1]) in enables - assert ("features", creates[1]) in enables - - -def test_teardown_deletes_release_tag_and_customer_property(): - from evals.seed import teardown - - plane = _TeardownPlane() - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "project_name": "EVAL x", - "workspace_objects": [ - {"kind": "release_tag", "id": "tag-tracked"}, - {"kind": "customer_property", "id": "prop-tracked"}, - ], - } - teardown(plane, ctx) - kinds = {k for k, _ in plane.deleted} - assert "release_tag" in kinds - assert "customer_property" in kinds - # Tracked ids deleted - assert ("release_tag", "tag-tracked") in plane.deleted - assert ("customer_property", "prop-tracked") in plane.deleted - - -def test_preclean_removes_stale_tag_and_property(): - from evals.seed import _preclean_ws3_workspace_artifacts - - deleted: list[tuple[str, str]] = [] - - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), - delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), - ) + plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": None}, + "project_id": None, + }, ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), - delete=lambda **kw: deleted.append(("prop", kw["property_id"])), + assert calls == [] + + def test_teardown_deletes_release_tag_and_customer_property(): + from evals.seed import teardown + + plane = _TeardownPlane() + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "project_name": "EVAL x", + "workspace_objects": [ + {"kind": "release_tag", "id": "tag-tracked"}, + {"kind": "customer_property", "id": "prop-tracked"}, + ], + } + teardown(plane, ctx) + kinds = {k for k, _ in plane.deleted} + assert "release_tag" in kinds + assert "customer_property" in kinds + # Tracked ids deleted + assert ("release_tag", "tag-tracked") in plane.deleted + assert ("customer_property", "prop-tracked") in plane.deleted + + test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown() + test_teardown_deletes_release_tag_and_customer_property() + + +def test_preclean_behaviours(): + def test_preclean_removes_stale_tag_and_property(): + from evals.seed import _preclean_ws3_workspace_artifacts + + deleted: list[tuple[str, str]] = [] + + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), + delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), + ) + ) + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), + delete=lambda **kw: deleted.append(("prop", kw["property_id"])), + ) ) - ) - - _preclean_ws3_workspace_artifacts(Plane(), "ws") - assert ("tag", "t-old") in deleted - assert ("prop", "p-old") in deleted + _preclean_ws3_workspace_artifacts(Plane(), "ws") + assert ("tag", "t-old") in deleted + assert ("prop", "p-old") in deleted -def test_preclean_delete_failure_raises_for_infra_seed(): - """Found artifact that cannot be deleted must raise (harness → infra_seed).""" - from evals.seed import _preclean_ws3_workspace_artifacts + def test_preclean_delete_failure_raises_for_infra_seed(): + from evals.seed import _preclean_ws3_workspace_artifacts - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), - delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), + class Plane: + releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), + delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), + ) ) - ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([]), - delete=lambda **kw: None, + customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) ) - ) - with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): - _preclean_ws3_workspace_artifacts(Plane(), "ws") + with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): + _preclean_ws3_workspace_artifacts(Plane(), "ws") + def test_preclean_empty_list_is_silent(): + from evals.seed import _preclean_ws3_workspace_artifacts -def test_preclean_empty_list_is_silent(): - from evals.seed import _preclean_ws3_workspace_artifacts + class Plane: + releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + customers = SimpleNamespace( + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None) + ) - class Plane: - releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) - customers = SimpleNamespace(properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) + _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise - _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise + test_preclean_removes_stale_tag_and_property() + test_preclean_delete_failure_raises_for_infra_seed() + test_preclean_empty_list_is_silent() -def test_l2_activity_gate_raises_when_empty(): - """Empty activities list after comments → TaskSkipped env:no-activity-worker.""" - from types import SimpleNamespace +def test_l2_activity_behaviours(): + def test_l2_activity_gate_raises_when_empty(): + from types import SimpleNamespace - from evals.seed import R5_TITLE, _gate_activity_worker - from evals.tasks.skip import TaskSkipped + from evals.seed import R5_TITLE, _gate_activity_worker + from evals.tasks.skip import TaskSkipped - class Plane: - work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) + class Plane: + work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - with pytest.raises(TaskSkipped, match="env:no-activity-worker"): - _gate_activity_worker(Plane(), "ws", ctx) + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + with pytest.raises(TaskSkipped, match="env:no-activity-worker"): + _gate_activity_worker(Plane(), "ws", ctx) + def test_l2_activity_gate_proceeds_when_nonempty(): + from types import SimpleNamespace -def test_l2_activity_gate_proceeds_when_nonempty(): - from types import SimpleNamespace + from evals.seed import R5_TITLE, _gate_activity_worker - from evals.seed import R5_TITLE, _gate_activity_worker + class Plane: + work_items = SimpleNamespace( + activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) + ) - class Plane: - work_items = SimpleNamespace( - activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) - ) + ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} + _gate_activity_worker(Plane(), "ws", ctx) # no raise - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - _gate_activity_worker(Plane(), "ws", ctx) # no raise + test_l2_activity_gate_raises_when_empty() + test_l2_activity_gate_proceeds_when_nonempty() -def test_create_project_retries_409_then_succeeds(monkeypatch): - attempts: list[str] = [] +def test_create_behaviours(monkeypatch): + def test_create_project_retries_409_then_succeeds(monkeypatch): + attempts: list[str] = [] - class FakeProjects: - def create(self, *, workspace_slug, data): - ident = data.identifier - attempts.append(ident) - if len(attempts) < 3: - raise HttpError("Project identifier already taken", 409) - return MagicMock(id="proj-ok", identifier=ident) + class FakeProjects: + def create(self, *, workspace_slug, data): + ident = data.identifier + attempts.append(ident) + if len(attempts) < 3: + raise HttpError("Project identifier already taken", 409) + return MagicMock(id="proj-ok", identifier=ident) - plane = MagicMock() - plane.projects = FakeProjects() + plane = MagicMock() + plane.projects = FakeProjects() - # Force deterministic retries after first collision. - suffixes = iter(["AAAA", "BBBB"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + # Force deterministic retries after first collision. + suffixes = iter(["AAAA", "BBBB"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - project = create_project_with_identifier_retry( - plane, - "ws", - name="EVAL abcd", - identifier_prefix="EV", - initial_suffix="DEAD", - ) - assert project.id == "proj-ok" - assert attempts[0] == "EVDEAD" - assert len(attempts) == 3 - assert attempts[1] != attempts[0] - assert attempts[2] != attempts[1] - assert attempts[1] == "EVAAAA" - assert attempts[2] == "EVBBBB" - - -def test_create_project_raises_after_max_409s(monkeypatch): - attempts: list[str] = [] - - class Always409: - def create(self, *, workspace_slug, data): - attempts.append(data.identifier) - raise HttpError("identifier already taken", 409) - - plane = MagicMock() - plane.projects = Always409() - suffixes = iter(["1111", "2222", "3333", "should-not-use"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( - plane, - "ws", - name="EVAL x", - identifier_prefix="EV", - initial_suffix="0000", - ) - assert ei.value.status_code == 409 - assert len(attempts) == 3 - assert attempts[0] == "EV0000" - assert attempts[1] != attempts[0] - assert attempts[1] == "EV1111" - assert attempts[2] == "EV2222" - - -def test_create_project_non_collision_error_does_not_retry(): - class Fail500: - def create(self, *, workspace_slug, data): - raise HttpError("server error", 500) - - plane = MagicMock() - plane.projects = Fail500() - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( + project = create_project_with_identifier_retry( plane, "ws", - name="EVAL x", + name="EVAL abcd", identifier_prefix="EV", - initial_suffix="0000", + initial_suffix="DEAD", ) - assert ei.value.status_code == 500 + assert project.id == "proj-ok" + assert attempts[0] == "EVDEAD" + assert len(attempts) == 3 + assert attempts[1] != attempts[0] + assert attempts[2] != attempts[1] + assert attempts[1] == "EVAAAA" + assert attempts[2] == "EVBBBB" + + def test_create_project_raises_after_max_409s(monkeypatch): + attempts: list[str] = [] + + class Always409: + def create(self, *, workspace_slug, data): + attempts.append(data.identifier) + raise HttpError("identifier already taken", 409) + + plane = MagicMock() + plane.projects = Always409() + suffixes = iter(["1111", "2222", "3333", "should-not-use"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 409 + assert len(attempts) == 3 + assert attempts[0] == "EV0000" + assert attempts[1] != attempts[0] + assert attempts[1] == "EV1111" + assert attempts[2] == "EV2222" + + def test_create_project_non_collision_error_does_not_retry(): + class Fail500: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + plane = MagicMock() + plane.projects = Fail500() + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="0000", + ) + assert ei.value.status_code == 500 + + with pytest.MonkeyPatch.context() as mp: + test_create_project_retries_409_then_succeeds(mp) + with pytest.MonkeyPatch.context() as mp: + test_create_project_raises_after_max_409s(mp) + test_create_project_non_collision_error_does_not_retry() def test_identifier_collision_requires_status_and_language(): @@ -700,106 +715,114 @@ def test_identifier_collision_requires_status_and_language(): assert is_identifier_collision(HttpError("identifier already taken", 500)) is False -def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): - projects = [ - SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), - SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), - SimpleNamespace(id="p3", name="Production", identifier="PROD"), - ] - delete_calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") - - def delete(self, **kwargs): - delete_calls.append(kwargs) - - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) - - rc = cleanup_mod.main([]) # dry-run - assert rc == 0 - assert delete_calls == [] - out = capsys.readouterr().out - assert "EVAL deadbeef" in out - assert "dry-run" in out - assert "Production" not in out # prefix filter - - -def test_cleanup_yes_deletes(monkeypatch, capsys): - projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] - delete_calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") - - def delete(self, **kwargs): - delete_calls.append(kwargs) - - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) - rc = cleanup_mod.main(["--yes"]) - assert rc == 0 - assert len(delete_calls) == 1 - assert delete_calls[0]["project_id"] == "p1" - - -def test_list_projects_with_prefix_filters(): - projects = [ - SimpleNamespace(id="1", name="EVAL a"), - SimpleNamespace(id="2", name="Other"), - SimpleNamespace(id="3", name="EVAL b"), - SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " - ] - calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - calls.append({"workspace_slug": workspace_slug, "params": params}) - assert params is not None - assert params.per_page == 100 - # SDK always populates next_cursor even on last page. - return SimpleNamespace( - results=projects, - next_page_results=False, - next_cursor="100:0:0", - ) - - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "3"] - assert len(calls) == 1 # one page only — no infinite loop on next_cursor - assert calls[0]["params"].cursor is None - - -def test_list_projects_two_page_pagination(): - page1 = [SimpleNamespace(id="1", name="EVAL one")] - page2 = [SimpleNamespace(id="2", name="EVAL two")] - seen_cursors: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - seen_cursors.append(getattr(params, "cursor", None)) - if params.cursor is None: +def test_cleanup_behaviours(monkeypatch, capsys): + def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): + projects = [ + SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), + SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), + SimpleNamespace(id="p3", name="Production", identifier="PROD"), + ] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + rc = cleanup_mod.main([]) # dry-run + assert rc == 0 + assert delete_calls == [] + out = capsys.readouterr().out + assert "EVAL deadbeef" in out + assert "dry-run" in out + assert "Production" not in out # prefix filter + + def test_cleanup_yes_deletes(monkeypatch, capsys): + projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + rc = cleanup_mod.main(["--yes"]) + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0]["project_id"] == "p1" + + with pytest.MonkeyPatch.context() as mp: + test_cleanup_dry_run_never_calls_delete(mp, capsys) + with pytest.MonkeyPatch.context() as mp: + test_cleanup_yes_deletes(mp, capsys) + + +def test_list_projects_behaviours(): + def test_list_projects_with_prefix_filters(): + projects = [ + SimpleNamespace(id="1", name="EVAL a"), + SimpleNamespace(id="2", name="Other"), + SimpleNamespace(id="3", name="EVAL b"), + SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " + ] + calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + calls.append({"workspace_slug": workspace_slug, "params": params}) + assert params is not None + assert params.per_page == 100 + # SDK always populates next_cursor even on last page. return SimpleNamespace( - results=page1, - next_page_results=True, + results=projects, + next_page_results=False, next_cursor="100:0:0", ) - assert params.cursor == "100:0:0" - return SimpleNamespace( - results=page2, - next_page_results=False, - next_cursor="200:0:0", - ) - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "2"] - assert seen_cursors == [None, "100:0:0"] + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "3"] + assert len(calls) == 1 # one page only — no infinite loop on next_cursor + assert calls[0]["params"].cursor is None + + def test_list_projects_two_page_pagination(): + page1 = [SimpleNamespace(id="1", name="EVAL one")] + page2 = [SimpleNamespace(id="2", name="EVAL two")] + seen_cursors: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + seen_cursors.append(getattr(params, "cursor", None)) + if params.cursor is None: + return SimpleNamespace( + results=page1, + next_page_results=True, + next_cursor="100:0:0", + ) + assert params.cursor == "100:0:0" + return SimpleNamespace( + results=page2, + next_page_results=False, + next_cursor="200:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "2"] + assert seen_cursors == [None, "100:0:0"] + + test_list_projects_with_prefix_filters() + test_list_projects_two_page_pagination() diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 119a501a..09603b4a 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -87,41 +87,55 @@ PINNED_SYNTHETIC_BATTERY = "3e9194740d73" -def test_catalog_includes_design_and_extras(): - ids = {t["id"] for t in TASKS} - assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" - assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" - assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" - assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" - assert len(TASKS) >= 20 +def test_catalog_behaviours(): + def test_catalog_includes_design_and_extras(): + ids = {t["id"] for t in TASKS} + assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" + assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" + assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" + assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" + assert len(TASKS) >= 20 + def test_catalog_id_order_is_pinned(): + assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER -def test_catalog_id_order_is_pinned(): - assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER + test_catalog_includes_design_and_extras() + test_catalog_id_order_is_pinned() -def test_get_tasks_all_and_filter(): - all_t = get_tasks(None) - assert len(all_t) == len(TASKS) - subset = get_tasks(["R1", "W9", "C2"]) - assert [t["id"] for t in subset] == ["R1", "W9", "C2"] +def test_get_tasks_behaviours(): + def test_get_tasks_all_and_filter(): + all_t = get_tasks(None) + assert len(all_t) == len(TASKS) + subset = get_tasks(["R1", "W9", "C2"]) + assert [t["id"] for t in subset] == ["R1", "W9", "C2"] + def test_get_tasks_unknown_exits(): + with pytest.raises(SystemExit): + get_tasks(["NOPE"]) -def test_get_tasks_unknown_exits(): - with pytest.raises(SystemExit): - get_tasks(["NOPE"]) + test_get_tasks_all_and_filter() + test_get_tasks_unknown_exits() -def test_task_schema_invariants(): - for t in TASKS: - assert t["id"] - assert isinstance(t["tags"], set) - assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS - assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] - assert isinstance(t["alternate_tools"], set) - assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] - assert callable(t["verify"]) - assert isinstance(t.get("needs"), set) +def test_task_behaviours(): + def test_task_schema_invariants(): + for t in TASKS: + assert t["id"] + assert isinstance(t["tags"], set) + assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS + assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] + assert isinstance(t["alternate_tools"], set) + assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] + assert callable(t["verify"]) + assert isinstance(t.get("needs"), set) + + def test_task_author_default(): + assert task_author({}) == "claude" + assert task_author({"author": "alice"}) == "alice" + + test_task_schema_invariants() + test_task_author_default() def test_debias_tasks_author(): @@ -165,85 +179,106 @@ def test_tasks_module_has_no_hardcoded_uuids(): assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) -def test_prompt_bind_strict_empty_raises(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import PromptBindError, format_task_prompt - - t = TASKS_BY_ID["I1"] - with pytest.raises(PromptBindError): - format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) - - -def test_prompt_bind_strict_exception_raises(): - from evals.tasks.prompts import PromptBindError, format_task_prompt - - def boom(_ctx): - raise RuntimeError("seed broken") - - task = { - "id": "X", - "prompt": "do {work_item_id}", - "prompt_bind": boom, - } - with pytest.raises(PromptBindError, match="prompt_bind failed"): - format_task_prompt(task, {"project_name": "P"}, strict=True) - - -def test_prompt_bind_dry_run_markers(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] - text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) - assert "" in text - assert "EVAL x" in text - - -def test_prompt_bind_strict_success(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] - text = format_task_prompt( - t, - {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, - strict=True, - ) - assert "uuid-abc" in text - assert "<" not in text - - -def test_task_author_default(): - assert task_author({}) == "claude" - assert task_author({"author": "alice"}) == "alice" - - -def test_battery_fingerprint_stable_and_sensitive(): - t1 = { - "id": "A", - "prompt": "p1 {project}", - "optimal_tools": {"b", "a"}, - "alternate_tools": {"c"}, - "optimal_calls": 2, - } - t2 = { - "id": "B", - "prompt": "p2", - "optimal_tools": {"x"}, - "alternate_tools": set(), - "optimal_calls": 1, - } - # Order of list must not matter (sorted by id). - h1 = battery_fingerprint([t2, t1]) - h2 = battery_fingerprint([t1, t2]) - assert h1 == h2 == PINNED_SYNTHETIC_BATTERY - assert len(h1) == 12 - - t1_edit = {**t1, "prompt": "p1 edited {project}"} - assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY - - # Subset of selected tasks → different fingerprint (documented ceiling). - assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY +def test_prompt_bind_behaviours(): + def test_prompt_bind_strict_empty_raises(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import PromptBindError, format_task_prompt + + t = TASKS_BY_ID["I1"] + with pytest.raises(PromptBindError): + format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) + + def test_prompt_bind_strict_exception_raises(): + from evals.tasks.prompts import PromptBindError, format_task_prompt + + def boom(_ctx): + raise RuntimeError("seed broken") + + task = { + "id": "X", + "prompt": "do {work_item_id}", + "prompt_bind": boom, + } + with pytest.raises(PromptBindError, match="prompt_bind failed"): + format_task_prompt(task, {"project_name": "P"}, strict=True) + + def test_prompt_bind_dry_run_markers(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) + assert "" in text + assert "EVAL x" in text + + def test_prompt_bind_strict_success(): + from evals.tasks.catalog import TASKS_BY_ID + from evals.tasks.prompts import format_task_prompt + + t = TASKS_BY_ID["I1"] + text = format_task_prompt( + t, + {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, + strict=True, + ) + assert "uuid-abc" in text + assert "<" not in text + + test_prompt_bind_strict_empty_raises() + test_prompt_bind_strict_exception_raises() + test_prompt_bind_dry_run_markers() + test_prompt_bind_strict_success() + + +def test_battery_fingerprint_behaviours(): + def test_battery_fingerprint_stable_and_sensitive(): + t1 = { + "id": "A", + "prompt": "p1 {project}", + "optimal_tools": {"b", "a"}, + "alternate_tools": {"c"}, + "optimal_calls": 2, + } + t2 = { + "id": "B", + "prompt": "p2", + "optimal_tools": {"x"}, + "alternate_tools": set(), + "optimal_calls": 1, + } + # Order of list must not matter (sorted by id). + h1 = battery_fingerprint([t2, t1]) + h2 = battery_fingerprint([t1, t2]) + assert h1 == h2 == PINNED_SYNTHETIC_BATTERY + assert len(h1) == 12 + + t1_edit = {**t1, "prompt": "p1 edited {project}"} + assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY + + # Subset of selected tasks → different fingerprint (documented ceiling). + assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY + + def test_battery_fingerprint_catalog_is_nonempty(): + from evals.tasks.catalog import TASKS + + fp = battery_fingerprint() + assert len(fp) == 12 + assert battery_fingerprint(list(TASKS)) == fp + + def test_battery_fingerprint_changes_with_new_debias_tasks(): + from evals.tasks.catalog import TASKS, TASKS_BY_ID + + full = battery_fingerprint() + without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] + assert without_debias, "pre-debias catalog should be non-empty" + reduced = battery_fingerprint(without_debias) + assert reduced != full + # Single new task also moves the hash relative to a reduced set. + assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced + + test_battery_fingerprint_stable_and_sensitive() + test_battery_fingerprint_catalog_is_nonempty() + test_battery_fingerprint_changes_with_new_debias_tasks() def test_revision_bump_changes_the_fingerprint_for_an_unchanged_catalog(): @@ -281,24 +316,3 @@ def test_fingerprint_records_the_revision_transition(): assert CATALOG_REVISION == 2 assert battery_fingerprint() == "4fb3a34a7231" - - -def test_battery_fingerprint_catalog_is_nonempty(): - from evals.tasks.catalog import TASKS - - fp = battery_fingerprint() - assert len(fp) == 12 - assert battery_fingerprint(list(TASKS)) == fp - - -def test_battery_fingerprint_changes_with_new_debias_tasks(): - """Adding I/L content must change the catalog fingerprint (content hash).""" - from evals.tasks.catalog import TASKS, TASKS_BY_ID - - full = battery_fingerprint() - without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] - assert without_debias, "pre-debias catalog should be non-empty" - reduced = battery_fingerprint(without_debias) - assert reduced != full - # Single new task also moves the hash relative to a reduced set. - assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py index 1a913b8d..5058ead8 100644 --- a/tests/evals/tasks/test_debias_verifiers.py +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -123,435 +123,238 @@ def __init__(self, n: int): self.work_items = SimpleNamespace(attachments=SimpleNamespace(list=lambda **kw: _Page(rows))) -def test_i1_untouched_urgent_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} - ok, note = await verify_i1(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) +BACKLOG = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") +DONE = SimpleNamespace(id="st-done", name="Done", group="completed") -def test_i1_wrong_item_high_target_still_urgent_fails(): - async def _go(): - # Right value on the wrong item; target remains urgent. - plane = _WIRetrievePlane( - by_id={ - "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), - "wi-other": SimpleNamespace(id="wi-other", priority="high"), - } - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} - ok, note = await verify_i1(plane, ctx, _run()) - assert ok is False, note - assert "urgent" in note or "high" in note +def _i1_ctx(): + return {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} - return asyncio.run(_go()) +def test_i1_passes_only_when_the_target_item_itself_changed(): + """Priority must land on the item the task names, not merely somewhere.""" -def test_i2_untouched_empty_final_text_fails(): async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[st], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("")) - assert ok is False, note + cases = [ + ( + "untouched: target still urgent", + _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}), + (), + ), + ( + "right value on the wrong item", + _WIRetrievePlane( + by_id={ + "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), + "wi-other": SimpleNamespace(id="wi-other", priority="high"), + } + ), + ("urgent", "high"), + ), + ] + for label, plane, expect_any in cases: + ok, note = await verify_i1(plane, _i1_ctx(), _run()) + assert ok is False, f"{label}: {note}" + if expect_any: + assert any(s in note for s in expect_any), f"{label}: {note}" return asyncio.run(_go()) -def test_i2_wrong_state_name_in_text_fails(): +def test_i2_requires_the_state_contract_to_name_the_real_state(): async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[ - st, - SimpleNamespace(id="st-done", name="Done", group="completed"), - ], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("Done")) - assert ok is False, note - - return asyncio.run(_go()) + def plane_for(states): + return _WIRetrievePlane(by_id={"wi-2": SimpleNamespace(id="wi-2", state=BACKLOG)}, states=states) - -def test_i2_exact_state_contract_passes(): - async def _go(): - st = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") - plane = _WIRetrievePlane( - by_id={"wi-2": SimpleNamespace(id="wi-2", state=st)}, - states=[st], - ) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} - ok, note = await verify_i2(plane, ctx, _run("state: Backlog")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_i3_untouched_not_on_cycle_fails(): - async def _go(): - plane = _I3Plane(["other-1", "other-2"]) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I3_TITLE: "footer-1"}, - "cycle_current_id": "cyc-1", - } - ok, note = await verify_i3(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i3_wrong_item_on_cycle_target_missing_fails(): - async def _go(): - plane = _I3Plane(["wrong-item"]) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I3_TITLE: "footer-1"}, - "cycle_current_id": "cyc-1", - } - ok, note = await verify_i3(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i4_untouched_no_label_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[])}) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I4_TITLE: "wi-4"}, - "labels": {"perf": "lab-perf"}, - } - ok, note = await verify_i4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i4_wrong_label_attached_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=[SimpleNamespace(id="lab-auth")])}) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "items": {I4_TITLE: "wi-4"}, - "labels": {"perf": "lab-perf"}, - } - ok, note = await verify_i4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i5_untouched_none_priority_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="none")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} - ok, note = await verify_i5(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_i5_wrong_value_high_fails(): - async def _go(): - plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority="high")}) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} - ok, note = await verify_i5(plane, ctx, _run()) - assert ok is False, note - assert "high" in note - - return asyncio.run(_go()) - - -def test_l1_untouched_no_worklog_fails(): - async def _go(): - plane = _L1Plane([]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l1_wrong_duration_120_fails(): - async def _go(): - plane = _L1Plane([120], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("Logged 120 minutes; summary ok.")) - assert ok is False, note - assert "90" in note + cases = [ + ("untouched: empty answer", [BACKLOG], "", False), + ("names a different state", [BACKLOG, DONE], "Done", False), + ("exact contract line", [BACKLOG], "state: Backlog", True), + ] + for label, states, text, want in cases: + ok, note = await verify_i2(plane_for(states), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_i3_fails_unless_the_target_item_is_on_the_cycle(): + async def _go(): + cases = [ + ("untouched: target never added", ["other-1", "other-2"]), + ("added the wrong item", ["wrong-item"]), + ] + for label, on_cycle in cases: + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(_I3Plane(on_cycle), ctx, _run()) + assert ok is False, f"{label}: {note}" return asyncio.run(_go()) -def test_l1_empty_summary_with_90m_log_fails(): - """Reviewer counterexample: 90m log present but final text empty → fail.""" - +def test_i4_requires_the_named_label_on_the_target(): async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("")) - assert ok is False, note - assert "logged-minutes" in note.lower() + cases = [ + ("untouched: no labels", []), + ("a different label attached", [SimpleNamespace(id="lab-auth")]), + ] + for label, labels in cases: + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=labels)}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, f"{label}: {note}" return asyncio.run(_go()) -def test_l1_one_hundred_ninety_minutes_fails(): - """Reviewer counterexample: English 'ninety' must not satisfy numeric duration.""" - +def test_i5_rejects_both_untouched_and_wrong_priority(): async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1( - plane, - ctx, - _run("Logged one hundred ninety minutes. Project summary looks fine."), - ) - assert ok is False, note - assert "duration" in note.lower() or "90" in note or "1.5" in note + cases = [ + ("untouched: priority none", "none", ()), + ("wrong value: high", "high", ("high",)), + ] + for label, priority, expect in cases: + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority=priority)}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, f"{label}: {note}" + for s in expect: + assert s in note, f"{label}: {note}" return asyncio.run(_go()) -def test_l1_prose_with_correct_facts_but_without_contract_fails(): - """Correct facts in prose do not satisfy the explicit output contract.""" +def test_l1_grades_the_duration_contract_not_the_prose(): + """Prose stating the right facts still fails: the format is part of the task. - async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("Logged 1.5 hours total.")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l1_ninety_minutes_of_work_fails_by_design(): - """Calibration: prose without contract lines fails by design.""" + Covers the counterexamples — English "ninety" is not a number, and a correct log with an empty answer. + """ async def _go(): - plane = _L1Plane([90], summary_ids=["wi-l1"]) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1(plane, ctx, _run("90 minutes of work")) - assert ok is False, note - assert "logged-minutes" in note.lower() - - return asyncio.run(_go()) - - -def test_l1_exact_duration_and_summary_contract_passes(): - async def _go(): + cases = [ + ("untouched: no worklog", [], None, "", False, ()), + ("wrong duration 120", [120], ["wi-l1"], "Logged 120 minutes; summary ok.", False, ("90",)), + ("90m logged but empty answer", [90], ["wi-l1"], "", False, ("logged-minutes",)), + ( + "English 'ninety' is not a number", + [90], + ["wi-l1"], + "Logged one hundred ninety minutes. Project summary looks fine.", + False, + (), + ), + ("correct facts, no contract", [90], ["wi-l1"], "Logged 1.5 hours total.", False, ()), + ("bare prose, no contract", [90], ["wi-l1"], "90 minutes of work", False, ("logged-minutes",)), + ( + "exact contract", + [90], + ["wi-l1"], + "logged-minutes: 90\nsummary-work-item-id: wi-l1", + True, + (), + ), + ] + for label, durations, summary_ids, text, want, expect in cases: + plane = _L1Plane(durations, summary_ids=summary_ids) + ok, note = await verify_l1(plane, dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + for s in expect: + assert s in note.lower(), f"{label}: {note}" + + # The 'ninety' case must name the duration it objected to. plane = _L1Plane([90], summary_ids=["wi-l1"]) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} - ok, note = await verify_l1( - plane, - ctx, - _run("logged-minutes: 90\nsummary-work-item-id: wi-l1"), + _, note = await verify_l1( + plane, dict(ctx), _run("Logged one hundred ninety minutes. Project summary looks fine.") ) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l2_untouched_empty_final_text_fails(): - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l2_contract_count_three_passes(): - """Contract line 'count: 3' truth=3 passes.""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("Saw some history.\ncount: 3")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l2_contract_count_two_fails_truth_three(): - """Contract 'count: 2' truth=3 fails.""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("count: 2")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l2_negative_contract_and_bare_fail_truth_three(): - """'-3' and 'count: -3' fail truth=3 (signed equality).""" - - async def _go(): - plane = _L2Plane(3) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok1, _ = await verify_l2(plane, ctx, _run("-3")) - ok2, _ = await verify_l2(plane, ctx, _run("count: -3")) - assert ok1 is False - assert ok2 is False + assert "duration" in note.lower() or "90" in note or "1.5" in note, note return asyncio.run(_go()) -def test_l2_prose_only_without_contract_fails_by_design(): - """By design: prose without 'count: N' (or bare int) fails — format is part of the task.""" - +def test_l2_counts_activities_through_the_contract_only(): async def _go(): - plane = _L2Plane(3) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} - ok, note = await verify_l2(plane, ctx, _run("There are 3 activities and some comment phrases.")) - assert ok is False, note + cases = [ + ("untouched: empty answer", "", False), + ("contract matches truth 3", "Saw some history.\ncount: 3", True), + ("contract says 2, truth 3", "count: 2", False), + ("bare negative", "-3", False), + ("negative contract", "count: -3", False), + ("prose without contract", "There are 3 activities and some comment phrases.", False), + ] + for label, text, want in cases: + ok, note = await verify_l2(_L2Plane(3), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" return asyncio.run(_go()) -def test_l3_untouched_no_tag_fails(): +def test_l3_requires_the_exact_release_tag_version(): async def _go(): - plane = _L3Plane([]) - ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) - assert ok is False, note + cases = [("untouched: no tags", [], ()), ("wrong version", ["v0.0.1", "other-rc"], (L3_TAG_VERSION,))] + for label, versions, expect in cases: + ok, note = await verify_l3(_L3Plane(versions), {"workspace_slug": "ws"}, _run()) + assert ok is False, f"{label}: {note}" + for s in expect: + assert s in note, f"{label}: {note}" return asyncio.run(_go()) -def test_l3_wrong_version_tag_fails(): - async def _go(): - plane = _L3Plane(["v0.0.1", "other-rc"]) - ok, note = await verify_l3(plane, {"workspace_slug": "ws"}, _run()) - assert ok is False, note - assert L3_TAG_VERSION in note - - return asyncio.run(_go()) - - -def test_l4_untouched_no_property_fails(): - async def _go(): - plane = _L4Plane(props=[], values={}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) +def test_l4_matches_the_property_on_name_type_and_value(): + """A URL-typed property whose name merely contains "Industry" must not satisfy it.""" - -def test_l4_right_property_wrong_value_fails(): async def _go(): - prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") - plane = _L4Plane(props=[prop], values={"prop-1": ["Startup"]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - assert L4_PROP_VALUE in note or "Startup" in note or "lack" in note - - return asyncio.run(_go()) - - -def test_l4_industry_url_type_with_enterprise_fails(): - """Reviewer counterexample: name contains Industry, URL type, value Enterprise → fail.""" - - async def _go(): - prop = SimpleNamespace( - id="prop-url", - display_name="Industry", # substring / wrong exact name - name="industry", - property_type="URL", - ) - plane = _L4Plane(props=[prop], values={"prop-url": [L4_PROP_VALUE]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l4_exact_text_enterprise_passes(): - async def _go(): - prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") - plane = _L4Plane(props=[prop], values={"prop-1": [L4_PROP_VALUE]}) - ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1"}} - ok, note = await verify_l4(plane, ctx, _run()) - assert ok is True, note - assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) - - return asyncio.run(_go()) - - -def test_l5_untouched_empty_final_text_fails(): - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_l5_bare_zero_passes(): - """Fallback: whole-answer bare '0' still passes truth=0.""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("0")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l5_multiline_ending_count_zero_passes(): - """Multi-line answer ending with 'count: 0' passes truth=0.""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5( - plane, - ctx, - _run("No files on this work item.\ncount: 0"), - ) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_l5_prose_only_without_contract_fails_by_design(): - """By design: prose without contract line fails (format instruction is part of the task).""" - - async def _go(): - plane = _L5Plane(0) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("There are 0 attachments.")) - assert ok is False, note + exact = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + loose = SimpleNamespace(id="prop-url", display_name="Industry", name="industry", property_type="URL") + cases = [ + ("untouched: no property", [], {}, False, ()), + ( + "right property, wrong value", + [exact], + {"prop-1": ["Startup"]}, + False, + (L4_PROP_VALUE, "Startup", "lack"), + ), + ("wrong name and type, right value", [loose], {"prop-url": [L4_PROP_VALUE]}, False, ()), + ("exact text property", [exact], {"prop-1": [L4_PROP_VALUE]}, True, ()), + ] + for label, props, values, want, expect_any in cases: + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} + ok, note = await verify_l4(_L4Plane(props=props, values=values), ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect_any: + assert any(s in note for s in expect_any), f"{label}: {note}" + if want: + assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) return asyncio.run(_go()) -def test_l5_wrong_contract_count_fails(): +def test_l5_accepts_a_zero_count_only_through_the_contract(): async def _go(): - plane = _L5Plane(0) ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} - ok, note = await verify_l5(plane, ctx, _run("count: 10")) - assert ok is False, note + cases = [ + ("untouched: empty answer", "", False), + ("bare zero as the whole answer", "0", True), + ("multiline ending in the contract", "No files on this work item.\ncount: 0", True), + ("prose without contract", "There are 0 attachments.", False), + ("contract with the wrong count", "count: 10", False), + ] + for label, text, want in cases: + ok, note = await verify_l5(_L5Plane(0), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" return asyncio.run(_go()) diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py index 08996b4a..538d7eca 100644 --- a/tests/evals/tasks/test_output_contracts.py +++ b/tests/evals/tasks/test_output_contracts.py @@ -71,23 +71,44 @@ def __init__(self, durations: list[int]): ) -def test_r2_written_number_prose_fails_and_count_contract_passes(): - async def _go(): - state = SimpleNamespace(id="started", name="Started", group="started") - items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] - plane = SimpleNamespace( - states=SimpleNamespace(list=lambda **kwargs: _Page([state])), - work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), - ) - ctx = {"workspace_slug": "ws", "project_id": "project"} +def test_r2_behaviours(): + def test_r2_written_number_prose_fails_and_count_contract_passes(): + async def _go(): + state = SimpleNamespace(id="started", name="Started", group="started") + items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] + plane = SimpleNamespace( + states=SimpleNamespace(list=lambda **kwargs: _Page([state])), + work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), + ) + ctx = {"workspace_slug": "ws", "project_id": "project"} - prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) - contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) + prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) + contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) - assert prose_ok is False - assert contract_ok is True, note + assert prose_ok is False + assert contract_ok is True, note - return asyncio.run(_go()) + return asyncio.run(_go()) + + def test_r2_rejects_a_count_that_disagrees_with_the_api(): + async def _go(): + from evals.tasks.read import verify_r2 as _vr2 + + urgent = [_item(str(i), "x", priority="urgent", state=SimpleNamespace(group="started")) for i in range(4)] + + class Plane: + work_items = SimpleNamespace(list=lambda **kw: _Page(urgent)) + states = SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) + ) + + ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) + assert ok is False, note + + return asyncio.run(_go()) + + test_r2_written_number_prose_fails_and_count_contract_passes() + test_r2_rejects_a_count_that_disagrees_with_the_api() def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): @@ -157,179 +178,79 @@ async def _go(): return asyncio.run(_go()) -def test_c2_correct_changelog_prose_without_contract_fails(): - async def _go(): - changelog = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." - prose = "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff." - - ok, _ = await verify_c2(object(), {"release_changelog_text": changelog}, _run(prose)) - - assert ok is False - - return asyncio.run(_go()) +CHANGELOG = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +R1_CTX = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], +} -def test_existing_r1_untouched_empty_text_fails(): +def test_r1_accepts_only_the_exact_state_contract(): async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_r1_wrong_state_in_text_fails(): - async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("Done")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_r1_exact_state_contract_passes(): - async def _go(): - plane = _R1Plane("In Progress") - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "r1_state_name": "In Progress", - "state_names": ["In Progress", "Done", "Backlog"], - } - ok, note = await verify_r1(plane, ctx, _run("state: In Progress")) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_existing_r2_wrong_count_in_text_fails(): - async def _go(): - # verify_r2 counts open urgent via SDK; text must match that count. - from evals.tasks.read import verify_r2 as _vr2 - - class Plane: - def __init__(self): - self.work_items = SimpleNamespace( - list=lambda **kw: _Page( - [ - _item("1", "a", priority="urgent", state=SimpleNamespace(group="started")), - _item("2", "b", priority="urgent", state=SimpleNamespace(group="started")), - _item("3", "c", priority="urgent", state=SimpleNamespace(group="started")), - _item("4", "d", priority="urgent", state=SimpleNamespace(group="started")), - ] - ) - ) - self.states = SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) - ) - - # If verifier only checks text against live count, empty/wrong text fails. - ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_w2_untouched_not_done_fails(): - async def _go(): - plane = _W2Plane("started", "In Progress") - ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note + cases = [ + ("untouched: empty answer", "", False), + ("names a different state", "Done", False), + ("exact contract line", "state: In Progress", True), + ] + for label, text, want in cases: + ok, note = await verify_r1(_R1Plane("In Progress"), dict(R1_CTX), _run(text)) + assert ok is want, f"{label}: {note}" return asyncio.run(_go()) -def test_existing_w2_wrong_cancelled_group_fails(): - async def _go(): - plane = _W2Plane("cancelled", "Cancelled") - ok, note = await verify_w2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note +def test_w2_requires_the_done_group_specifically(): + """Cancelled is also terminal, so a verifier keying on "not started" would pass it.""" - return asyncio.run(_go()) - - -def test_existing_w4_untouched_still_triage_fails(): async def _go(): - plane = _W4Plane("triage") - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is False, note + cases = [ + ("untouched: still in progress", "started", "In Progress"), + ("cancelled, not done", "cancelled", "Cancelled"), + ] + for label, group, name in cases: + ok, note = await verify_w2(_W2Plane(group, name), {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, f"{label}: {note}" return asyncio.run(_go()) -def test_existing_w4_wrong_name_needs_review_fails(): +def test_w4_requires_the_label_renamed_to_the_exact_target(): async def _go(): - plane = _W4Plane("needs-review") ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is False, note + for label, name in [("untouched: still triage", "triage"), ("renamed to something else", "needs-review")]: + ok, note = await verify_w4(_W4Plane(name), dict(ctx), _run()) + assert ok is False, f"{label}: {note}" return asyncio.run(_go()) -def test_existing_w8_untouched_no_log_fails(): +def test_w8_requires_a_log_of_exactly_the_asked_duration(): async def _go(): - plane = _W8Plane([]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note + for label, durations in [("untouched: no log", []), ("wrong duration", [60])]: + ok, note = await verify_w8(_W8Plane(durations), {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, f"{label}: {note}" return asyncio.run(_go()) -def test_existing_w8_wrong_duration_fails(): - async def _go(): - plane = _W8Plane([60]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_c2_untouched_empty_text_fails(): - async def _go(): - ctx = {"release_changelog_text": "Changelog entry one: OAuth login hardening."} - ok, note = await verify_c2(object(), ctx, _run("")) - assert ok is False, note - - return asyncio.run(_go()) +def test_c2_grades_the_release_contract_not_correct_prose(): + """Prose naming the right release and entries still fails; the format is the task.""" - -def test_existing_c2_wrong_release_name_fails(): async def _go(): - ok, note = await verify_c2( - object(), - {"release_changelog_text": "Changelog entry one: OAuth login hardening."}, - _run("Release 9.9.9 shipped nothing useful."), - ) - assert ok is False, note - - return asyncio.run(_go()) - - -def test_existing_c2_exact_release_and_shipped_contract_passes(): - async def _go(): - ok, note = await verify_c2( - object(), - { - "release_changelog_text": ( - "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." - ) - }, - _run("release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff"), - ) - assert ok is True, note + cases = [ + ("untouched: empty answer", "", False), + ("wrong release name", "Release 9.9.9 shipped nothing useful.", False), + ("correct facts as prose", "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff.", False), + ( + "exact contract", + "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff", + True, + ), + ] + for label, text, want in cases: + ok, note = await verify_c2(object(), {"release_changelog_text": CHANGELOG}, _run(text)) + assert ok is want, f"{label}: {note}" return asyncio.run(_go()) diff --git a/tests/evals/tasks/test_verifiers.py b/tests/evals/tasks/test_verifiers.py index 51592d28..a944487d 100644 --- a/tests/evals/tasks/test_verifiers.py +++ b/tests/evals/tasks/test_verifiers.py @@ -72,36 +72,39 @@ def _links_list(self, **kw): return _Page([SimpleNamespace(url=u) for u in self._urls]) -def test_f1_w7_blocked_by_only_does_not_pass(): - async def _go(): - """tgt id only in blocked_by (reverse) must FAIL — not pass via str(dump).""" - plane = _W7Plane( - { - "blocking": [], - "blocked_by": [{"id": "tgt-1"}], # reverse direction only - } - ) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_w7(plane, ctx, _run()) - assert ok is False, note - assert "blocking" in note.lower() or "no blocking" in note.lower() - assert "wrong direction" in note or "tgt-1" in note +def test_f1_w7_behaviours(): + def test_f1_w7_blocked_by_only_does_not_pass(): + async def _go(): + """tgt id only in blocked_by (reverse) must FAIL — not pass via str(dump).""" + plane = _W7Plane( + { + "blocking": [], + "blocked_by": [{"id": "tgt-1"}], # reverse direction only + } + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w7(plane, ctx, _run()) + assert ok is False, note + assert "blocking" in note.lower() or "no blocking" in note.lower() + assert "wrong direction" in note or "tgt-1" in note - return asyncio.run(_go()) + return asyncio.run(_go()) + def test_f1_w7_blocking_passes(): + async def _go(): + plane = _W7Plane({"blocking": [{"id": "tgt-1"}], "blocked_by": []}) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w7(plane, ctx, _run()) + assert ok is True, note -def test_f1_w7_blocking_passes(): - async def _go(): - plane = _W7Plane({"blocking": [{"id": "tgt-1"}], "blocked_by": []}) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_w7(plane, ctx, _run()) - assert ok is True, note + # --------------------------------------------------------------------------- + # F2 W6 — seeded past end_date alone must NOT pass as closed + # --------------------------------------------------------------------------- - # --------------------------------------------------------------------------- - # F2 W6 — seeded past end_date alone must NOT pass as closed - # --------------------------------------------------------------------------- + return asyncio.run(_go()) - return asyncio.run(_go()) + test_f1_w7_blocked_by_only_does_not_pass() + test_f1_w7_blocking_passes() class _W6Plane: @@ -124,102 +127,37 @@ def _list_wi(self, **kw): return _Page([_item(f"i{i}", n) for i, n in enumerate(self._s13)]) -def test_f2_w6_seeded_past_end_alone_fails(): - async def _go(): - """Sprint 12 still at seed end_date (today-14) with no archive → not closed.""" - seed_end = (date.today() - timedelta(days=14)).isoformat() - plane = _W6Plane(past_end=seed_end, sprint13_names=["Inventory count goes negative under load"]) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "cycle_past_id": "c12", - "cycle_current_id": "c13", - "cycle_past_seed_end_date": seed_end, - "w6_unfinished_titles": ["Inventory count goes negative under load"], - "cycles": {CYCLE_PAST: "c12"}, - } - ok, note = await verify_w6(plane, ctx, _run()) - assert ok is False, note - assert "not closed" in note.lower() or "Sprint 12 not closed" in note - - return asyncio.run(_go()) - +def test_f2_w6_closes_only_on_a_real_end_date_signal(): + """The API returns end_date as a timestamp, so whole-string comparison to today never + matched — which silently left the transfer side effect as the only way to pass.""" -def test_f2_w6_end_date_today_as_timestamp_passes_close(): async def _go(): - """The API returns end_date as a timestamp, not a bare date. - - Sprint 12 closed today comes back as 'T00:00:00Z'; comparing the - whole string to today's date never matches, which silently killed the - complete_cycle close signal and left the transfer side effect - (progress_snapshot) as the only way to pass. - """ today = date.today().isoformat() + past = (date.today() - timedelta(days=14)).isoformat() + tomorrow = (date.today() + timedelta(days=1)).isoformat() titles = ["Inventory count goes negative under load"] - plane = _W6Plane(past_end=f"{today}T00:00:00Z", sprint13_names=titles) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "cycle_past_id": "c12", - "cycle_current_id": "c13", - "cycle_past_seed_end_date": (date.today() + timedelta(days=1)).isoformat(), - "w6_unfinished_titles": titles, - } - ok, note = await verify_w6(plane, ctx, _run()) - assert ok is True, note - assert "end_date" in note - - return asyncio.run(_go()) - - -def test_f2_w6_open_seed_end_tomorrow_alone_fails(): - async def _go(): - """A no-op agent leaves Sprint 12 ending tomorrow → not closed.""" - seed_end = (date.today() + timedelta(days=1)).isoformat() - titles = ["Inventory count goes negative under load"] - plane = _W6Plane(past_end=f"{seed_end}T00:00:00Z", sprint13_names=titles) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "cycle_past_id": "c12", - "cycle_current_id": "c13", - "cycle_past_seed_end_date": seed_end, - "w6_unfinished_titles": titles, - } - ok, note = await verify_w6(plane, ctx, _run()) - assert ok is False, note - assert "not closed" in note.lower() - - return asyncio.run(_go()) - - -def test_f2_w6_end_date_today_passes_close(): - async def _go(): - today = date.today().isoformat() - plane = _W6Plane( - past_end=today, - sprint13_names=[ - "Inventory count goes negative under load", - "Tooltip clipped inside modal dialog", - ], - ) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "cycle_past_id": "c12", - "cycle_current_id": "c13", - "cycle_past_seed_end_date": (date.today() - timedelta(days=14)).isoformat(), - "w6_unfinished_titles": [ - "Inventory count goes negative under load", - "Tooltip clipped inside modal dialog", - ], - } - ok, note = await verify_w6(plane, ctx, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F3 W5 — 404 without archived list entry is NOT archive - # --------------------------------------------------------------------------- + two = [*titles, "Tooltip clipped inside modal dialog"] + cases = [ + ("still at the seeded past end, never closed", past, past, titles, False, "not closed"), + ("no-op agent: still ends tomorrow", f"{tomorrow}T00:00:00Z", tomorrow, titles, False, "not closed"), + ("closed today, returned as a timestamp", f"{today}T00:00:00Z", tomorrow, titles, True, "end_date"), + ("closed today, returned as a bare date", today, past, two, True, ""), + ] + for label, end_date, seed_end, names, want, expect in cases: + plane = _W6Plane(past_end=end_date, sprint13_names=names) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": seed_end, + "w6_unfinished_titles": names, + "cycles": {CYCLE_PAST: "c12"}, + } + ok, note = await verify_w6(plane, ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect: + assert expect in note.lower() or expect in note, f"{label}: {note}" return asyncio.run(_go()) @@ -247,48 +185,29 @@ def _list_archived(self, **kw): return _Page([_item(i, f"n-{i}") for i in self._archived_ids]) -def test_f3_w5_deleted_404_without_archive_fails(): - async def _go(): - """All items 404 and archived list empty → fail (deletes ≠ archive).""" - ids = ["m1", "m2", "m3"] - plane = _W5Plane(retrieve_map={}, archived_ids=[]) # all 404, none archived - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "module_completed_ids": ids, - } - ok, note = await verify_w5(plane, ctx, _run()) - assert ok is False, note - assert "not archived" in note - - return asyncio.run(_go()) - +def test_f3_w5_accepts_archive_but_not_deletion(): + """A 404 alone is not evidence of archiving — deleting every item would also 404.""" -def test_f3_w5_404_present_in_archived_passes(): async def _go(): ids = ["m1", "m2", "m3"] - plane = _W5Plane(retrieve_map={}, archived_ids=ids) - ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ids} - ok, note = await verify_w5(plane, ctx, _run()) - assert ok is True, note - - return asyncio.run(_go()) - - -def test_f3_w5_archived_at_on_retrieve_passes(): - async def _go(): - ids = ["m1"] - plane = _W5Plane( - retrieve_map={"m1": SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z")}, - archived_ids=[], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ids} - ok, note = await verify_w5(plane, ctx, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F4 C1 — must link exact R1 item; acme* / random links fail - # --------------------------------------------------------------------------- + archived_at = SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z") + cases = [ + ("all 404, nothing archived", _W5Plane(retrieve_map={}, archived_ids=[]), ids, False, "not archived"), + ("404 but present in the archived list", _W5Plane(retrieve_map={}, archived_ids=ids), ids, True, ""), + ( + "archived_at on retrieve", + _W5Plane(retrieve_map={"m1": archived_at}, archived_ids=[]), + ["m1"], + True, + "", + ), + ] + for label, plane, module_ids, want, expect in cases: + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": module_ids} + ok, note = await verify_w5(plane, ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect: + assert expect in note, f"{label}: {note}" return asyncio.run(_go()) @@ -310,53 +229,44 @@ def __init__( self.work_items = SimpleNamespace(list=lambda **kw: _Page(project_items)) -def test_f4_c1_wrong_customer_name_fails(): - async def _go(): - plane = _C1Plane( - customers=[SimpleNamespace(id="c1", name="Acme Industries")], - requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], - linked=[SimpleNamespace(id="wi-r1")], - project_items=[_item("wi-r1", R1_TITLE)], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} - ok, note = await verify_c1(plane, ctx, _run()) - assert ok is False, note - assert CUSTOMER_NAME in note - - return asyncio.run(_go()) - +def test_f4_c1_requires_the_named_customer_linked_to_the_named_item(): + """Both ends are checked: a lookalike customer name and a link to any other item fail.""" -def test_f4_c1_linked_other_item_not_r1_fails(): async def _go(): - plane = _C1Plane( - customers=[SimpleNamespace(id="c1", name=CUSTOMER_NAME)], - requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], - linked=[SimpleNamespace(id="wi-other")], # not R1 - project_items=[_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_c1(plane, ctx, _run()) - assert ok is False, note - assert "not linked" in note or "wi-r1" in note - - return asyncio.run(_go()) - - -def test_f4_c1_exact_r1_link_passes(): - async def _go(): - plane = _C1Plane( - customers=[SimpleNamespace(id="c1", name=CUSTOMER_NAME)], - requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], - linked=[SimpleNamespace(id="wi-r1")], - project_items=[_item("wi-r1", R1_TITLE)], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_c1(plane, ctx, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F5 W3 — comment must contain 'contrast tokens' - # --------------------------------------------------------------------------- + request = SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME) + cases = [ + ( + "customer name is a lookalike", + [SimpleNamespace(id="c1", name="Acme Industries")], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + False, + (CUSTOMER_NAME,), + ), + ( + "linked to a different item", + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-other")], + [_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], + False, + ("not linked", "wi-r1"), + ), + ( + "exact customer and item", + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + True, + (), + ), + ] + for label, customers, linked, project_items, want, expect_any in cases: + plane = _C1Plane(customers=customers, requests=[request], linked=linked, project_items=project_items) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} + ok, note = await verify_c1(plane, ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect_any: + assert any(s in note for s in expect_any), f"{label}: {note}" return asyncio.run(_go()) @@ -369,32 +279,19 @@ def __init__(self, comments: list[Any]): ) -def test_f5_w3_unrelated_comment_fails(): +def test_f5_w3_matches_the_phrase_in_either_comment_field(): async def _go(): - plane = _W3Plane([SimpleNamespace(comment_html="

lgtm

", comment_stripped="lgtm")]) - ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "contrast tokens" in note - - return asyncio.run(_go()) - - -def test_f5_w3_phrase_in_html_passes(): - async def _go(): - plane = _W3Plane( - [ - SimpleNamespace( - comment_html="

Reviewed contrast tokens — needs design pass

", - comment_stripped="Reviewed contrast tokens — needs design pass", - ) - ] - ) - ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F7 W4 — seeded triage id is authoritative - # --------------------------------------------------------------------------- + html = "

Reviewed contrast tokens — needs design pass

" + cases = [ + ("unrelated comment", "

lgtm

", "lgtm", False, "contrast tokens"), + ("phrase present", html, "Reviewed contrast tokens — needs design pass", True, ""), + ] + for label, comment_html, stripped, want, expect in cases: + plane = _W3Plane([SimpleNamespace(comment_html=comment_html, comment_stripped=stripped)]) + ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is want, f"{label}: {note}" + if expect: + assert expect in note, f"{label}: {note}" return asyncio.run(_go()) @@ -411,37 +308,32 @@ def _retrieve(self, **kw): return self._by_id[lid] -def test_f7_w4_wrong_label_renamed_triage_id_unchanged_fails(): - async def _go(): - """Name-scan would see needs-triage, but seeded triage id still named triage.""" - plane = _W4Plane( - by_id={"triage-id": SimpleNamespace(id="triage-id", name="triage")}, - listed=[ - SimpleNamespace(id="triage-id", name="triage"), - SimpleNamespace(id="other", name="needs-triage"), # wrong label renamed - ], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is False, note - assert "triage-id" in note - - return asyncio.run(_go()) - +def test_f7_w4_follows_the_seeded_label_id_not_the_name(): + """A name scan would accept a different label renamed to the target.""" -def test_f7_w4_seeded_id_renamed_passes(): async def _go(): - plane = _W4Plane( - by_id={"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, - listed=[SimpleNamespace(id="triage-id", name="needs-triage")], - ) - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(plane, ctx, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F6 S3 — required OPTION must not pass as TEXT - # --------------------------------------------------------------------------- + cases = [ + ( + "decoy label carries the new name", + {"triage-id": SimpleNamespace(id="triage-id", name="triage")}, + [SimpleNamespace(id="triage-id", name="triage"), SimpleNamespace(id="other", name="needs-triage")], + False, + "triage-id", + ), + ( + "the seeded label itself was renamed", + {"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, + [SimpleNamespace(id="triage-id", name="needs-triage")], + True, + "", + ), + ] + for label, by_id, listed, want, expect in cases: + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = await verify_w4(_W4Plane(by_id=by_id, listed=listed), ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect: + assert expect in note, f"{label}: {note}" return asyncio.run(_go()) @@ -459,45 +351,48 @@ def __init__(self, *, props: list[Any], types: list[Any] | None = None, workspac self.workspace_work_item_types = SimpleNamespace(list=lambda **kw: []) -def test_f6_s3_required_option_does_not_pass(): - async def _go(): - plane = _S3Plane( - props=[ - SimpleNamespace( - id="p1", - display_name="Severity", - property_type="OPTION", - is_required=True, - ) - ] - ) - ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "TEXT" in note - - return asyncio.run(_go()) - - -def test_f6_s3_required_text_passes(): - async def _go(): - plane = _S3Plane( - props=[ - SimpleNamespace( - id="p1", - display_name="Impact summary", - property_type="TEXT", - is_required=True, - ) - ] - ) - ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F9 S3 — workspace types found via get_features probe (not seed flag) - # --------------------------------------------------------------------------- - - return asyncio.run(_go()) +def test_f6_s3_behaviours(): + def test_f6_s3_required_option_does_not_pass(): + async def _go(): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name="Severity", + property_type="OPTION", + is_required=True, + ) + ] + ) + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "TEXT" in note + + return asyncio.run(_go()) + + def test_f6_s3_required_text_passes(): + async def _go(): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name="Impact summary", + property_type="TEXT", + is_required=True, + ) + ] + ) + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # F9 S3 — workspace types found via get_features probe (not seed flag) + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + test_f6_s3_required_option_does_not_pass() + test_f6_s3_required_text_passes() def test_f9_s3_workspace_type_via_features_probe(): @@ -529,38 +424,38 @@ async def _go(): return asyncio.run(_go()) -def test_f8_r3_due_date_clamped_to_iso_week(): - """today+2d on Sat/Sun must not leave the week — replicate seed formula.""" - for weekday in range(7): # 0=Mon … 6=Sun - # Build a fixed "today" with that weekday relative to a known Monday. - # 2026-08-10 is a Monday. - monday = date(2026, 8, 10) - today = monday + timedelta(days=weekday) - days_to_week_end = 6 - today.weekday() - due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) - # Sunday of that week - week_end = today + timedelta(days=days_to_week_end) - week_start = today - timedelta(days=today.weekday()) - assert week_start <= due <= week_end, f"weekday={weekday} due={due}" - # Specifically: Sat/Sun must not go past Sunday - if weekday >= 5: - assert due <= week_end - assert due == week_end or due == today # Sun→today, Sat→Sun - - -def test_f8_seed_r3_due_date_function_matches(): - """Import seed's computation path by re-running the formula against weekends.""" - # Saturday - sat = date(2026, 8, 15) # known Saturday - assert sat.weekday() == 5 - days_to_week_end = 6 - sat.weekday() - due = min(sat + timedelta(days=2), sat + timedelta(days=days_to_week_end)) - assert due == date(2026, 8, 16) # Sunday, not Monday 17 - # Sunday - sun = date(2026, 8, 16) - days_to_week_end = 6 - sun.weekday() - due = min(sun + timedelta(days=2), sun + timedelta(days=days_to_week_end)) - assert due == sun +def test_f8_behaviours(): + def test_f8_r3_due_date_clamped_to_iso_week(): + for weekday in range(7): # 0=Mon … 6=Sun + # Build a fixed "today" with that weekday relative to a known Monday. + # 2026-08-10 is a Monday. + monday = date(2026, 8, 10) + today = monday + timedelta(days=weekday) + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + # Sunday of that week + week_end = today + timedelta(days=days_to_week_end) + week_start = today - timedelta(days=today.weekday()) + assert week_start <= due <= week_end, f"weekday={weekday} due={due}" + # Specifically: Sat/Sun must not go past Sunday + if weekday >= 5: + assert due <= week_end + assert due == week_end or due == today # Sun→today, Sat→Sun + + def test_f8_seed_r3_due_date_function_matches(): + sat = date(2026, 8, 15) # known Saturday + assert sat.weekday() == 5 + days_to_week_end = 6 - sat.weekday() + due = min(sat + timedelta(days=2), sat + timedelta(days=days_to_week_end)) + assert due == date(2026, 8, 16) # Sunday, not Monday 17 + # Sunday + sun = date(2026, 8, 16) + days_to_week_end = 6 - sun.weekday() + due = min(sun + timedelta(days=2), sun + timedelta(days=days_to_week_end)) + assert due == sun + + test_f8_r3_due_date_clamped_to_iso_week() + test_f8_seed_r3_due_date_function_matches() class _W8Plane: @@ -571,59 +466,68 @@ def __init__(self, durations: list[int]): ) -def test_minor_w8_480_minutes_fails(): - async def _go(): - plane = _W8Plane([480]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "120" in note - - return asyncio.run(_go()) - - -def test_minor_w8_exactly_120_passes(): - async def _go(): - plane = _W8Plane([120]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # Minor R3 — all titles required; count alone insufficient - # --------------------------------------------------------------------------- - - return asyncio.run(_go()) +def test_minor_behaviours(): + def test_minor_w8_behaviours(): + def test_minor_w8_480_minutes_fails(): + async def _go(): + plane = _W8Plane([480]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, note + assert "120" in note + + return asyncio.run(_go()) + + def test_minor_w8_exactly_120_passes(): + async def _go(): + plane = _W8Plane([120]) + ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + # --------------------------------------------------------------------------- + # Minor R3 — all titles required; count alone insufficient + # --------------------------------------------------------------------------- + + return asyncio.run(_go()) + + test_minor_w8_480_minutes_fails() + test_minor_w8_exactly_120_passes() + + def test_minor_r3_behaviours(): + def test_minor_r3_count_without_titles_fails(): + async def _go(): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + run = {"final_text": "There are 2 items due this week.", "calls": []} + ok, note = await verify_r3( + object(), + {"r3_due_titles": titles, "r3_due_count": 2}, + run, + ) + assert ok is False, note + assert "item contract" in note.lower() + return asyncio.run(_go()) -def test_minor_r3_count_without_titles_fails(): - async def _go(): - titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] - run = {"final_text": "There are 2 items due this week.", "calls": []} - ok, note = await verify_r3( - object(), - {"r3_due_titles": titles, "r3_due_count": 2}, - run, - ) - assert ok is False, note - assert "item contract" in note.lower() - - return asyncio.run(_go()) + def test_minor_r3_exact_item_contract_passes_in_any_order(): + async def _go(): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + run = { + "final_text": ("item: Onboarding email template stale\nitem: Webhook secret rotation docs missing"), + "calls": [], + } + ok, note = await verify_r3( + object(), + {"r3_due_titles": titles, "r3_due_count": 2}, + run, + ) + assert ok is True, note + return asyncio.run(_go()) -def test_minor_r3_exact_item_contract_passes_in_any_order(): - async def _go(): - titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] - run = { - "final_text": ("item: Onboarding email template stale\nitem: Webhook secret rotation docs missing"), - "calls": [], - } - ok, note = await verify_r3( - object(), - {"r3_due_titles": titles, "r3_due_count": 2}, - run, - ) - assert ok is True, note + test_minor_r3_count_without_titles_fails() + test_minor_r3_exact_item_contract_passes_in_any_order() - return asyncio.run(_go()) + test_minor_w8_behaviours() + test_minor_r3_behaviours() class _S5Plane: @@ -653,55 +557,28 @@ def __init__( ) -def test_s5_only_cycles_enabled_fails(): - """Two-of-three: cycles on, worklogs off, customers on → fail.""" +def test_s5_requires_all_three_features_not_a_majority(): + """Every two-of-three combination must fail; the task asks for all three.""" async def _go(): - plane = _S5Plane(cycle_view=True, time_tracking=False, customers=True) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "is_time_tracking_enabled" in note - - return asyncio.run(_go()) - - -def test_s5_only_worklogs_enabled_fails(): - async def _go(): - plane = _S5Plane(cycle_view=False, time_tracking=True, customers=True) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "cycle_view" in note - - return asyncio.run(_go()) - - -def test_s5_workspace_customers_on_but_project_flags_off_fails(): - """Workspace customers alone is not enough — project gates must also pass.""" - - async def _go(): - plane = _S5Plane(cycle_view=False, time_tracking=False, customers=True) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "cycle_view" in note - assert "is_time_tracking_enabled" in note - - return asyncio.run(_go()) - - -def test_s5_project_flags_on_but_customers_off_fails(): - """Two-of-three: project ok, workspace customers off → fail.""" - - async def _go(): - plane = _S5Plane(cycle_view=True, time_tracking=True, customers=False, features_cycles=True) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "customers" in note - - return asyncio.run(_go()) - + cases = [ + ("cycles+customers, worklogs off", True, False, True, False, ("is_time_tracking_enabled",)), + ("worklogs+customers, cycles off", False, True, True, False, ("cycle_view",)), + ("customers only", False, False, True, False, ("cycle_view", "is_time_tracking_enabled")), + ("project flags only, customers off", True, True, False, True, ("customers",)), + ] + for label, cycle_view, tracking, customers, features_cycles, expect in cases: + plane = _S5Plane( + cycle_view=cycle_view, + time_tracking=tracking, + customers=customers, + features_cycles=features_cycles, + ) + ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, f"{label}: {note}" + for s in expect: + assert s in note, f"{label}: {note}" -def test_s5_all_three_enabled_passes(): - async def _go(): plane = _S5Plane(cycle_view=True, time_tracking=True, customers=True, features_cycles=True) ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) assert ok is True, note diff --git a/tests/evals/test_cli.py b/tests/evals/test_cli.py index 87b29564..36c1ca3e 100644 --- a/tests/evals/test_cli.py +++ b/tests/evals/test_cli.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - import pytest from evals import cli as run_mod @@ -37,47 +35,53 @@ EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features -def test_cmd_list_prints_all_task_ids(capsys): - rc = cmd_list() - assert rc == 0 - out = capsys.readouterr().out - for tid in DESIGN_IDS | EXTRA_IDS: - assert tid in out - - -def test_cmd_dry_run_all_tasks(capsys): - rc = cmd_dry_run(list(TASKS)) - assert rc == 0 - out = capsys.readouterr().out - assert "Seed plan:" in out - for tid in ("R1", "W9", "S4", "C2", "R7"): - assert f"=== {tid} ===" in out - - -def test_parse_args_list(): - a = parse_args(["--list", "--label", "candidate-build"]) - assert a.list is True - assert a.label == "candidate-build" - assert parse_args(["--list"]).label == "local" - - -def test_parse_args_accepts_driver(): - a = parse_args(["--driver", "claude-cli", "--dry-run"]) - assert a.driver == "claude-cli" - b = parse_args(["--dry-run"]) - assert b.driver == "api" - assert b.model == "standard" - assert b.provider == "anthropic" - assert b.record_result_payloads is False - c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) - assert c.record_result_payloads is True - - -def test_parse_args_resume_and_canary(): - a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) - assert a.resume == "evals/output/x.jsonl" - b = run_mod.parse_args(["--canary", "--tasks", "R1"]) - assert b.canary is True +def test_cmd_behaviours(capsys): + def test_cmd_list_prints_all_task_ids(capsys): + rc = cmd_list() + assert rc == 0 + out = capsys.readouterr().out + for tid in DESIGN_IDS | EXTRA_IDS: + assert tid in out + + def test_cmd_dry_run_all_tasks(capsys): + rc = cmd_dry_run(list(TASKS)) + assert rc == 0 + out = capsys.readouterr().out + assert "Seed plan:" in out + for tid in ("R1", "W9", "S4", "C2", "R7"): + assert f"=== {tid} ===" in out + + test_cmd_list_prints_all_task_ids(capsys) + test_cmd_dry_run_all_tasks(capsys) + + +def test_parse_args_behaviours(): + def test_parse_args_list(): + a = parse_args(["--list", "--label", "candidate-build"]) + assert a.list is True + assert a.label == "candidate-build" + assert parse_args(["--list"]).label == "local" + + def test_parse_args_accepts_driver(): + a = parse_args(["--driver", "claude-cli", "--dry-run"]) + assert a.driver == "claude-cli" + b = parse_args(["--dry-run"]) + assert b.driver == "api" + assert b.model == "standard" + assert b.provider == "anthropic" + assert b.record_result_payloads is False + c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) + assert c.record_result_payloads is True + + def test_parse_args_resume_and_canary(): + a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) + assert a.resume == "evals/output/x.jsonl" + b = run_mod.parse_args(["--canary", "--tasks", "R1"]) + assert b.canary is True + + test_parse_args_list() + test_parse_args_accepts_driver() + test_parse_args_resume_and_canary() def test_model_tiers_resolve_per_driver_and_provider(): @@ -108,30 +112,35 @@ def test_non_tier_model_strings_pass_through_unchanged(driver, model): assert resolve_model_for_driver(driver, model) == model -def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): - with pytest.raises(ValueError, match=r"opencode models"): - resolve_model_for_driver("opencode-cli", "standard") - - -def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path: Path, capsys): - out = tmp_path / "must-not-exist.jsonl" - - rc = eval_main( - [ - "--driver", - "opencode-cli", - "--model", - "standard", - "--tasks", - "R1", - "--out", - str(out), - ] - ) - - assert rc == 2 - assert "explicit provider/model ID" in capsys.readouterr().err - assert out.exists() is False +def test_unmapped_behaviours(tmp_path, capsys): + def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): + with pytest.raises(ValueError, match=r"opencode models"): + resolve_model_for_driver("opencode-cli", "standard") + + def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path, capsys): + out = tmp_path / "must-not-exist.jsonl" + + rc = eval_main( + [ + "--driver", + "opencode-cli", + "--model", + "standard", + "--tasks", + "R1", + "--out", + str(out), + ] + ) + + assert rc == 2 + assert "explicit provider/model ID" in capsys.readouterr().err + assert out.exists() is False + + test_unmapped_opencode_tier_fails_with_explicit_model_guidance() + _d1 = tmp_path / "test_unmapped_tier_cli_error_is_loud_and_prevents_run" + _d1.mkdir() + test_unmapped_tier_cli_error_is_loud_and_prevents_run(_d1, capsys) def test_tier_mapping_is_scoped_to_cli_provider(): diff --git a/tests/evals/test_docs.py b/tests/evals/test_docs.py index 79300920..4a4d945e 100644 --- a/tests/evals/test_docs.py +++ b/tests/evals/test_docs.py @@ -13,24 +13,25 @@ DESIGN = (REPO_ROOT / "evals" / "DESIGN.md").read_text() -def test_the_flag_server_is_documented_as_optional(): - """A gated capability skips its task; it does not stop the battery.""" - assert "FEATURE_FLAG_SERVER_BASE_URL" in README, "the option should still be documented" - index = README.index("FEATURE_FLAG_SERVER_BASE_URL") - paragraph = README[max(0, index - 200) : index + 500] - assert "not** required" in paragraph or "Optionally" in paragraph, ( - "the flag server stopped being a prerequisite when the seeders learned to skip; " - "the runbook must not tell people otherwise" - ) - - -def test_the_plan_gate_skip_reason_is_documented(): - assert "env:plan-gated:" in README - - -def test_the_fingerprint_revision_is_documented(): - """Anyone comparing two batteries needs to know a revision bump can move the hash.""" - assert "CATALOG_REVISION" in README +def test_the_behaviours(): + def test_the_flag_server_is_documented_as_optional(): + assert "FEATURE_FLAG_SERVER_BASE_URL" in README, "the option should still be documented" + index = README.index("FEATURE_FLAG_SERVER_BASE_URL") + paragraph = README[max(0, index - 200) : index + 500] + assert "not** required" in paragraph or "Optionally" in paragraph, ( + "the flag server stopped being a prerequisite when the seeders learned to skip; " + "the runbook must not tell people otherwise" + ) + + def test_the_plan_gate_skip_reason_is_documented(): + assert "env:plan-gated:" in README + + def test_the_fingerprint_revision_is_documented(): + assert "CATALOG_REVISION" in README + + test_the_flag_server_is_documented_as_optional() + test_the_plan_gate_skip_reason_is_documented() + test_the_fingerprint_revision_is_documented() def test_design_still_states_the_skip_contract_the_seeders_now_implement(): diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 97d7e63e..b7aa95db 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -100,173 +100,503 @@ def _write_fake_server(path: Path) -> Path: return path -def test_proxy_records_tools_call_and_exit_code(tmp_path: Path): - server = _write_fake_server(tmp_path / "fake_server.py") - sidecar = tmp_path / "side.jsonl" - cmd = [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ] - # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. - client_in = ( - json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) - + "\n" - + json.dumps( +def test_proxy_behaviours(tmp_path): + def test_proxy_records_tools_call_and_exit_code(tmp_path): + server = _write_fake_server(tmp_path / "fake_server.py") + sidecar = tmp_path / "side.jsonl" + cmd = [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ] + # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. + client_in = ( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {"project": "P"}}, + } + ) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "boom", "arguments": {}}, + } + ) + + "\n" + + "NOT_JSON_LINE\n" + ) + proc = subprocess.run( + cmd, + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7 # child exit propagated + # Byte-faithful: unparsed line and JSON responses appear on stdout. + out = proc.stdout.decode("utf-8", errors="replace") + assert "NOT_JSON_LINE" in out + assert "list_work_items" in out or "ok:list_work_items" in out + + rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert len(call_rows) == 2 + assert call_rows[0]["tool"] == "list_work_items" + assert call_rows[0]["args"] == {"project": "P"} + assert call_rows[0]["is_error"] is False + assert call_rows[0]["result_chars"] > 0 + assert call_rows[0]["seq"] == 1 + assert call_rows[1]["tool"] == "boom" + assert call_rows[1]["is_error"] is True + assert meta["unparsed_lines"] >= 1 + assert meta["relayed_lines"] >= 3 + + def test_proxy_byte_faithful_child_receives_exact_bytes(tmp_path): + received = tmp_path / "received.bin" + echo_server = tmp_path / "echo_server.py" + echo_server.write_text( + textwrap.dedent( + f""" + import sys + data = sys.stdin.buffer.read() + open({str(received)!r}, "wb").write(data) + # Still answer initialize-ish so proxy drains cleanly + for line in data.splitlines(keepends=True): + if not line.strip(): + continue + try: + import json + msg = json.loads(line) + except Exception: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + continue + if msg.get("id") is not None: + sys.stdout.buffer.write( + (json.dumps({{"jsonrpc": "2.0", "id": msg["id"], "result": {{}}}}) + "\\n").encode() + ) + sys.stdout.buffer.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "s.jsonl" + # Deliberately non-canonical JSON spacing — re-serialization would change it. + payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(echo_server), + ], + input=payload, + capture_output=True, + cwd=str(REPO), + timeout=10, + ) + assert proc.returncode == 0 + assert received.read_bytes() == payload + + def test_proxy_main_requires_command(): + with pytest.raises(SystemExit): + proxy_main(["--log", "/tmp/x.jsonl"]) + + def test_proxy_exits_when_child_dies_first(tmp_path): + server = tmp_path / "die_soon.py" + server.write_text( + textwrap.dedent( + """ + import sys, time + # Emit nothing and exit quickly; leave proxy client stdin open. + time.sleep(0.15) + sys.exit(3) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + t0 = __import__("time").monotonic() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + # Keep stdin open (do not close) so the stdin pump blocks on readline; + # the proxy must still notice child death and exit. + deadline = SHUTDOWN_DEADLINE_S + 5.0 + try: + rc = proc.wait(timeout=deadline) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + pytest.fail(f"proxy hung >{deadline}s after child exit") + elapsed = __import__("time").monotonic() - t0 + # Must finish well under the hang window (not wait the full drain). + assert elapsed < deadline + # Child's exit code (3) should propagate; tolerate signal map if the + # runtime reaps oddly, but meta must still be present. + assert rc in (3, 128 + 3) or rc == 3 + assert sidecar.is_file() + text = sidecar.read_text(encoding="utf-8") + assert "proxy_meta" in text + # Prefer exact child code when available + if rc not in (3, 128 + 3): + # At least ensure we did not hang; surface stderr for diagnosis. + err = (proc.stderr.read() if proc.stderr else b"").decode() + assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.stdin: + try: + proc.stdin.close() + except Exception: + pass + + def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path): + server = tmp_path / "echo_once.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {"content": [], "isError": False}, + }) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign_cwd" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) + # Drop any ambient PYTHONPATH pollution by putting repo first. + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "ping", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), # foreign cwd — must still import evals.proxy + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "ping" + + def test_proxy_child_env_pythonpath_clean(tmp_path): + server = tmp_path / "check_env.py" + server.write_text( + textwrap.dedent( + f""" + import json, os, sys + root = {str(REPO)!r} + pp = os.environ.get("PYTHONPATH", "") + parts = [p for p in pp.split(os.pathsep) if p] + bad = root in parts + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({{ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {{"content": [{{"type": "text", "text": "bad=" + str(bad)}}], "isError": False}}, + }}) + "\\n") + sys.stdout.flush() + sys.exit(0 if not bad else 9) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(__import__("os").environ)) + assert str(REPO) in env.get("PYTHONPATH", "") + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "envcheck", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + assert b"bad=False" in proc.stdout + + def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path): + import os + import signal + import time + + server = tmp_path / "echo_server.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: + continue + mid = msg.get("id") + if mid is not None: + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + leader_script = tmp_path / "cli_leader.py" + leader_script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + sidecar = Path({str(sidecar)!r}) + server = Path({str(server)!r}) + # Spawn proxy in our process group (no start_new_session on child). + # proxy main() will os.setsid() and detach. + proxy = subprocess.Popen( + [ + sys.executable, "-m", "evals.proxy", + "--log", str(sidecar), + "--", + sys.executable, str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # Give setsid a moment, then write a tools/call and keep stdin open briefly. + time.sleep(0.4) + if proxy.stdin: + req = ( + '{{"jsonrpc":"2.0","id":1,"method":"tools/call",' + '"params":{{"name":"t","arguments":{{}}}}}}\\n' + ) + proxy.stdin.write(req.encode()) + proxy.stdin.flush() + # Stay alive as group leader until killed by the test harness. + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Leader is a process-group leader (like run_cli_subprocess). + env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} + leader = subprocess.Popen( + [sys.executable, str(leader_script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(REPO), + env=env, + ) + try: + # Wait until proxy has started (sidecar created) and setsid likely done. + boot = time.monotonic() + 5.0 + while time.monotonic() < boot: + if sidecar.is_file(): + break + time.sleep(0.05) + time.sleep(0.5) # allow setsid + optional tools/call + # SIGKILL the CLI process group — must NOT kill the detached proxy. + try: + os.killpg(leader.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + leader.kill() + leader.wait(timeout=1.0) + + # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. + deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 + meta_seen = False + while time.monotonic() < deadline: + if sidecar.is_file(): + text = sidecar.read_text(encoding="utf-8") + if "proxy_meta" in text: + meta_seen = True + break + time.sleep(0.1) + assert meta_seen, ( + f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" + ) + rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass + + _d0 = tmp_path / "test_proxy_records_tools_call_and_exit_code" + _d0.mkdir() + test_proxy_records_tools_call_and_exit_code(_d0) + _d1 = tmp_path / "test_proxy_byte_faithful_child_receives_exact_bytes" + _d1.mkdir() + test_proxy_byte_faithful_child_receives_exact_bytes(_d1) + test_proxy_main_requires_command() + _d3 = tmp_path / "test_proxy_exits_when_child_dies_first" + _d3.mkdir() + test_proxy_exits_when_child_dies_first(_d3) + _d4 = tmp_path / "test_proxy_from_foreign_cwd_with_pythonpath" + _d4.mkdir() + test_proxy_from_foreign_cwd_with_pythonpath(_d4) + _d5 = tmp_path / "test_proxy_child_env_pythonpath_clean" + _d5.mkdir() + test_proxy_child_env_pythonpath_clean(_d5) + _d6 = tmp_path / "test_proxy_survives_cli_group_kill_and_writes_meta" + _d6.mkdir() + test_proxy_survives_cli_group_kill_and_writes_meta(_d6) + + +def test_sidecar_behaviours(tmp_path): + def test_sidecar_recorder_unit(tmp_path): + rec = SidecarRecorder(tmp_path / "a.jsonl") + rec.on_client_message( { "jsonrpc": "2.0", - "id": 2, + "id": 9, "method": "tools/call", - "params": {"name": "list_work_items", "arguments": {"project": "P"}}, + "params": {"name": "t", "arguments": {"a": 1}}, } ) - + "\n" - + json.dumps( + rec.on_server_message({"jsonrpc": "2.0", "id": 9, "result": {"content": [], "isError": False}}) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") + raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] + raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") + assert len(calls) == 1 + assert calls[0]["tool"] == "t" + assert calls[0]["args"] == {"a": 1} + assert calls[0]["origin"] == "plane" + assert "result_text" not in calls[0] + assert "result_text" not in raw_call + assert rec.finalized is True + + def test_sidecar_result_payload_round_trips_only_when_enabled(tmp_path): + path = tmp_path / "payload.jsonl" + rec = SidecarRecorder(path, record_result_payloads=True) + rec.on_client_message( { "jsonrpc": "2.0", "id": 3, "method": "tools/call", - "params": {"name": "boom", "arguments": {}}, + "params": {"name": "find_work_items", "arguments": {}}, } ) - + "\n" - + "NOT_JSON_LINE\n" - ) - proc = subprocess.run( - cmd, - input=client_in.encode("utf-8"), - capture_output=True, - cwd=str(REPO), - timeout=15, - ) - assert proc.returncode == 7 # child exit propagated - # Byte-faithful: unparsed line and JSON responses appear on stdout. - out = proc.stdout.decode("utf-8", errors="replace") - assert "NOT_JSON_LINE" in out - assert "list_work_items" in out or "ok:list_work_items" in out - - rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] - call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] - meta = next(r for r in rows if r.get("row_type") == "proxy_meta") - assert len(call_rows) == 2 - assert call_rows[0]["tool"] == "list_work_items" - assert call_rows[0]["args"] == {"project": "P"} - assert call_rows[0]["is_error"] is False - assert call_rows[0]["result_chars"] > 0 - assert call_rows[0]["seq"] == 1 - assert call_rows[1]["tool"] == "boom" - assert call_rows[1]["is_error"] is True - assert meta["unparsed_lines"] >= 1 - assert meta["relayed_lines"] >= 3 - - -def test_proxy_byte_faithful_child_receives_exact_bytes(tmp_path: Path): - """Child sees the exact request bytes the client sent (no re-serialization).""" - received = tmp_path / "received.bin" - echo_server = tmp_path / "echo_server.py" - echo_server.write_text( - textwrap.dedent( - f""" - import sys - data = sys.stdin.buffer.read() - open({str(received)!r}, "wb").write(data) - # Still answer initialize-ish so proxy drains cleanly - for line in data.splitlines(keepends=True): - if not line.strip(): - continue - try: - import json - msg = json.loads(line) - except Exception: - sys.stdout.buffer.write(line) - sys.stdout.buffer.flush() - continue - if msg.get("id") is not None: - sys.stdout.buffer.write( - (json.dumps({{"jsonrpc": "2.0", "id": msg["id"], "result": {{}}}}) + "\\n").encode() - ) - sys.stdout.buffer.flush() - """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "s.jsonl" - # Deliberately non-canonical JSON spacing — re-serialization would change it. - payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(echo_server), - ], - input=payload, - capture_output=True, - cwd=str(REPO), - timeout=10, - ) - assert proc.returncode == 0 - assert received.read_bytes() == payload - - -def test_sidecar_recorder_unit(tmp_path: Path): - rec = SidecarRecorder(tmp_path / "a.jsonl") - rec.on_client_message( - { - "jsonrpc": "2.0", - "id": 9, - "method": "tools/call", - "params": {"name": "t", "arguments": {"a": 1}}, - } - ) - rec.on_server_message({"jsonrpc": "2.0", "id": 9, "result": {"content": [], "isError": False}}) - rec.write_meta() - calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") - raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] - raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") - assert len(calls) == 1 - assert calls[0]["tool"] == "t" - assert calls[0]["args"] == {"a": 1} - assert calls[0]["origin"] == "plane" - assert "result_text" not in calls[0] - assert "result_text" not in raw_call - assert rec.finalized is True - - -def test_sidecar_result_payload_round_trips_only_when_enabled(tmp_path: Path): - path = tmp_path / "payload.jsonl" - rec = SidecarRecorder(path, record_result_payloads=True) - rec.on_client_message( - { - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": {"name": "find_work_items", "arguments": {}}, - } - ) - result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} - rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) - rec.write_meta() + result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} + rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) + rec.write_meta() + + expected_text = json.dumps(result, default=str, ensure_ascii=False) + raw_call = next( + row + for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) + if row.get("row_type") != "proxy_meta" + ) + assert raw_call["result_text"] == expected_text + calls = load_proxy_sidecar_calls(path) + assert calls[0]["result_text"] == expected_text + assert calls[0]["result_chars"] == len(expected_text) - expected_text = json.dumps(result, default=str, ensure_ascii=False) - raw_call = next( - row - for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) - if row.get("row_type") != "proxy_meta" - ) - assert raw_call["result_text"] == expected_text - calls = load_proxy_sidecar_calls(path) - assert calls[0]["result_text"] == expected_text - assert calls[0]["result_chars"] == len(expected_text) + _d0 = tmp_path / "test_sidecar_recorder_unit" + _d0.mkdir() + test_sidecar_recorder_unit(_d0) + _d1 = tmp_path / "test_sidecar_result_payload_round_trips_only_when_enabled" + _d1.mkdir() + test_sidecar_result_payload_round_trips_only_when_enabled(_d1) def test_append_after_finalize_is_dropped(tmp_path: Path): @@ -359,11 +689,6 @@ def test_reap_timeout_floor_when_deadline_exhausted(): assert reap_timeout(future, floor=0.1) >= 4.0 -def test_proxy_main_requires_command(): - with pytest.raises(SystemExit): - proxy_main(["--log", "/tmp/x.jsonl"]) - - def test_server_initiated_request_does_not_pop_pending(tmp_path: Path): """Server message with method+id must not complete a tools/call pending slot.""" rec = SidecarRecorder(tmp_path / "s.jsonl") @@ -429,132 +754,6 @@ def test_write_all_fd_loops_on_short_writes(tmp_path: Path): assert got == payload -def test_proxy_exits_when_child_dies_first(tmp_path: Path): - """Child exits while parent stdin is still open — proxy must not hang.""" - server = tmp_path / "die_soon.py" - server.write_text( - textwrap.dedent( - """ - import sys, time - # Emit nothing and exit quickly; leave proxy client stdin open. - time.sleep(0.15) - sys.exit(3) - """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - t0 = __import__("time").monotonic() - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=str(REPO), - ) - try: - # Keep stdin open (do not close) so the stdin pump blocks on readline; - # the proxy must still notice child death and exit. - deadline = SHUTDOWN_DEADLINE_S + 5.0 - try: - rc = proc.wait(timeout=deadline) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - pytest.fail(f"proxy hung >{deadline}s after child exit") - elapsed = __import__("time").monotonic() - t0 - # Must finish well under the hang window (not wait the full drain). - assert elapsed < deadline - # Child's exit code (3) should propagate; tolerate signal map if the - # runtime reaps oddly, but meta must still be present. - assert rc in (3, 128 + 3) or rc == 3 - assert sidecar.is_file() - text = sidecar.read_text(encoding="utf-8") - assert "proxy_meta" in text - # Prefer exact child code when available - if rc not in (3, 128 + 3): - # At least ensure we did not hang; surface stderr for diagnosis. - err = (proc.stderr.read() if proc.stderr else b"").decode() - assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" - finally: - if proc.poll() is None: - proc.kill() - proc.wait() - if proc.stdin: - try: - proc.stdin.close() - except Exception: - pass - - -def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path: Path): - """proxy + plane path must resolve when cwd is a temp dir (OpenCode case).""" - server = tmp_path / "echo_once.py" - server.write_text( - textwrap.dedent( - """ - import json, sys - line = sys.stdin.readline() - msg = json.loads(line) - sys.stdout.write(json.dumps({ - "jsonrpc": "2.0", - "id": msg["id"], - "result": {"content": [], "isError": False}, - }) + "\\n") - sys.stdout.flush() - """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - foreign = tmp_path / "foreign_cwd" - foreign.mkdir() - env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) - # Drop any ambient PYTHONPATH pollution by putting repo first. - assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) - client_in = ( - json.dumps( - { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "ping", "arguments": {}}, - } - ) - + "\n" - ) - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - input=client_in.encode(), - capture_output=True, - cwd=str(foreign), # foreign cwd — must still import evals.proxy - env=env, - timeout=15, - ) - assert proc.returncode == 0, proc.stderr.decode() - calls = load_proxy_sidecar_calls(sidecar) - assert len(calls) == 1 - assert calls[0]["tool"] == "ping" - - def test_process_buffer_partial_line_and_multi_line_chunk(tmp_path: Path): """Partial line stays buffered; two lines in one chunk both process.""" import os @@ -648,67 +847,6 @@ def test_scrub_child_pythonpath_removes_repo(): assert "PYTHONPATH" not in only -def test_proxy_child_env_pythonpath_clean(tmp_path: Path): - """Real MCP child must not inherit the monorepo PYTHONPATH entry.""" - server = tmp_path / "check_env.py" - server.write_text( - textwrap.dedent( - f""" - import json, os, sys - root = {str(REPO)!r} - pp = os.environ.get("PYTHONPATH", "") - parts = [p for p in pp.split(os.pathsep) if p] - bad = root in parts - line = sys.stdin.readline() - msg = json.loads(line) - sys.stdout.write(json.dumps({{ - "jsonrpc": "2.0", - "id": msg["id"], - "result": {{"content": [{{"type": "text", "text": "bad=" + str(bad)}}], "isError": False}}, - }}) + "\\n") - sys.stdout.flush() - sys.exit(0 if not bad else 9) - """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - foreign = tmp_path / "foreign" - foreign.mkdir() - env = ensure_proxy_pythonpath(dict(__import__("os").environ)) - assert str(REPO) in env.get("PYTHONPATH", "") - client_in = ( - json.dumps( - { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "envcheck", "arguments": {}}, - } - ) - + "\n" - ) - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - input=client_in.encode(), - capture_output=True, - cwd=str(foreign), - env=env, - timeout=15, - ) - assert proc.returncode == 0, proc.stderr.decode() - assert b"bad=False" in proc.stdout - - def test_rapid_response_pairing(tmp_path: Path): """Record-before-forward: fast child responses must pair with requests (no unmatched). @@ -914,129 +1052,3 @@ def test_bounded_shutdown_wall_clock(tmp_path: Path): assert elapsed < SHUTDOWN_DEADLINE_S + 2.0 rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] assert rows[-1].get("row_type") == "proxy_meta" - - -def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path: Path): - """Proxy os.setsid() detaches from the CLI process group. - - Simulate: CLI process-group leader spawns proxy as a child (same group); - proxy main() calls setsid and leaves the group; SIGKILL the CLI group; - proxy still finalizes proxy_meta within the shutdown deadline. - """ - import os - import signal - import time - - server = tmp_path / "echo_server.py" - server.write_text( - textwrap.dedent( - """ - import json, sys - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except Exception: - continue - mid = msg.get("id") - if mid is not None: - sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n") - sys.stdout.flush() - """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - leader_script = tmp_path / "cli_leader.py" - leader_script.write_text( - textwrap.dedent( - f""" - import os, subprocess, sys, time - from pathlib import Path - sidecar = Path({str(sidecar)!r}) - server = Path({str(server)!r}) - # Spawn proxy in our process group (no start_new_session on child). - # proxy main() will os.setsid() and detach. - proxy = subprocess.Popen( - [ - sys.executable, "-m", "evals.proxy", - "--log", str(sidecar), - "--", - sys.executable, str(server), - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - # Give setsid a moment, then write a tools/call and keep stdin open briefly. - time.sleep(0.4) - if proxy.stdin: - req = ( - '{{"jsonrpc":"2.0","id":1,"method":"tools/call",' - '"params":{{"name":"t","arguments":{{}}}}}}\\n' - ) - proxy.stdin.write(req.encode()) - proxy.stdin.flush() - # Stay alive as group leader until killed by the test harness. - time.sleep(9999) - """ - ), - encoding="utf-8", - ) - - # Leader is a process-group leader (like run_cli_subprocess). - env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} - leader = subprocess.Popen( - [sys.executable, str(leader_script)], - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - cwd=str(REPO), - env=env, - ) - try: - # Wait until proxy has started (sidecar created) and setsid likely done. - boot = time.monotonic() + 5.0 - while time.monotonic() < boot: - if sidecar.is_file(): - break - time.sleep(0.05) - time.sleep(0.5) # allow setsid + optional tools/call - # SIGKILL the CLI process group — must NOT kill the detached proxy. - try: - os.killpg(leader.pid, signal.SIGKILL) - except ProcessLookupError: - pass - try: - leader.wait(timeout=2.0) - except subprocess.TimeoutExpired: - leader.kill() - leader.wait(timeout=1.0) - - # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. - deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 - meta_seen = False - while time.monotonic() < deadline: - if sidecar.is_file(): - text = sidecar.read_text(encoding="utf-8") - if "proxy_meta" in text: - meta_seen = True - break - time.sleep(0.1) - assert meta_seen, ( - f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" - ) - rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] - assert rows[-1].get("row_type") == "proxy_meta" - finally: - if leader.poll() is None: - try: - os.killpg(leader.pid, signal.SIGKILL) - except Exception: - leader.kill() - try: - leader.wait(timeout=2.0) - except Exception: - pass diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py index cc522253..9d3708fb 100644 --- a/tests/evals/test_results.py +++ b/tests/evals/test_results.py @@ -167,165 +167,162 @@ def test_api_driver_maps_every_legacy_row_field(): assert row["provider_stop_reason"] == "fake_done" -def test_agent_run_dict_keeps_action_arg(): - run = AgentRun( - calls=[ - {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, - {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, - ], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - d = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=lambda t, o, a: "out_of_set", - ) - assert d["calls"][0]["action"] == "create" - assert "action" not in d["calls"][1] - - -def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): - """F1: ToolSearch must not inflate out_of_set or num_calls.""" - run = AgentRun( - calls=[ - normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), - ], - client_tool_calls=[ - normalize_tool_call("ToolSearch", {"query": "work items"}), - ], - final_text="done", - usage={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_cost_usd": 0.29, - "modelUsage": { - "claude-sonnet": { - "inputTokens": 10, - "outputTokens": 865, - "cacheReadInputTokens": 250433, - "cacheCreationInputTokens": 33838, - "costUSD": 0.29, - } +def test_agent_run_behaviours(): + def test_agent_run_dict_keeps_action_arg(): + run = AgentRun( + calls=[ + {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, + {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, + ], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + d = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=lambda t, o, a: "out_of_set", + ) + assert d["calls"][0]["action"] == "create" + assert "action" not in d["calls"][1] + + def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): + run = AgentRun( + calls=[ + normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), + ], + client_tool_calls=[ + normalize_tool_call("ToolSearch", {"query": "work items"}), + ], + final_text="done", + usage={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_cost_usd": 0.29, + "modelUsage": { + "claude-sonnet": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.29, + } + }, }, - }, - usage_total={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_input_tokens_including_cache": 10 + 250433 + 33838, - "total_cost_usd": 0.29, - "source": "modelUsage", - }, - stopped_reason="end_turn", - usage_scope="run", - call_source="transcript", - hit_max_turns=False, - wall_time_s=1.5, - ) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate={"get_work_item"}, - classify=classify_call, - ) - assert out["num_calls"] == 1 - assert out["out_of_set_calls"] == 0 - assert out["calls"][0]["class"] == "optimal" - assert out["client_tool_call_count"] == 1 - assert out["client_tool_calls"][0]["tool"] == "ToolSearch" - # F2: cum_input_tokens null — not the misleading uncached-only 10 - assert out["cum_input_tokens"] is None - assert out["cum_input_tokens_reason"] - assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 - assert out["usage_per_iteration"] == [] - assert out["calls"][0]["result_tokens"] == 0 - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True - assert "result_tokens_skipped_reason" not in out - - -def test_agent_run_hit_max_maps_to_hit_max_iterations(): - run = AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - hit_max_turns=True, - call_source="json", - ) - out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) - assert out["hit_max_iterations"] is True - assert out["stop_reason"] == "max_turns" - - -def test_agent_run_to_harness_dict_does_not_guess_usage_total(): - """Generic row mapping must not invent usage_total from a vendor usage dict. - - Drivers own normalization (ClaudeCliDriver via normalize_claude_usage, - CodexCliDriver builds its own). Missing usage_total stays None. - """ - run = AgentRun( - calls=[], - final_text="ok", - usage={ - "input_tokens": 5000, - "output_tokens": 200, - # Codex-ish shape — not Claude modelUsage. A Claude rebuild would - # silently produce a wrong / empty total if reintroduced. - "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, - }, - usage_total=None, - stopped_reason="completed", - usage_scope="run", - call_source="stream", - ) - out = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=classify_call, - ) - assert out["usage"] == run.usage - assert out["usage_total"] is None - - -def test_agent_run_to_harness_propagates_proxy_fields(): - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {"q": "a"}, - "origin": "plane", - "is_error": True, - "result_chars": 99, - "duration_ms": 42, - } - ], - final_text="x", - usage=None, - stopped_reason="end_turn", - call_source="proxy", - usage_scope="run", - ) - d = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=lambda t, o, a: "optimal", - ) - assert d["calls"][0]["is_error"] is True - assert d["calls"][0]["result_chars"] == 99 - assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) - assert d["calls"][0]["result_tokens_estimated"] is True - assert d["result_tokens_estimated"] is True - assert d["calls"][0]["duration_ms"] == 42 - assert d["errored_calls"] == 1 + usage_total={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_input_tokens_including_cache": 10 + 250433 + 33838, + "total_cost_usd": 0.29, + "source": "modelUsage", + }, + stopped_reason="end_turn", + usage_scope="run", + call_source="transcript", + hit_max_turns=False, + wall_time_s=1.5, + ) + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate={"get_work_item"}, + classify=classify_call, + ) + assert out["num_calls"] == 1 + assert out["out_of_set_calls"] == 0 + assert out["calls"][0]["class"] == "optimal" + assert out["client_tool_call_count"] == 1 + assert out["client_tool_calls"][0]["tool"] == "ToolSearch" + # F2: cum_input_tokens null — not the misleading uncached-only 10 + assert out["cum_input_tokens"] is None + assert out["cum_input_tokens_reason"] + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["usage_per_iteration"] == [] + assert out["calls"][0]["result_tokens"] == 0 + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + assert "result_tokens_skipped_reason" not in out + + def test_agent_run_hit_max_maps_to_hit_max_iterations(): + run = AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + hit_max_turns=True, + call_source="json", + ) + out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) + assert out["hit_max_iterations"] is True + assert out["stop_reason"] == "max_turns" + + def test_agent_run_to_harness_dict_does_not_guess_usage_total(): + run = AgentRun( + calls=[], + final_text="ok", + usage={ + "input_tokens": 5000, + "output_tokens": 200, + # Codex-ish shape — not Claude modelUsage. A Claude rebuild would + # silently produce a wrong / empty total if reintroduced. + "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, + }, + usage_total=None, + stopped_reason="completed", + usage_scope="run", + call_source="stream", + ) + out = agent_run_to_harness_dict( + run, + optimal=set(), + alternate=set(), + classify=classify_call, + ) + assert out["usage"] == run.usage + assert out["usage_total"] is None + + def test_agent_run_to_harness_propagates_proxy_fields(): + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {"q": "a"}, + "origin": "plane", + "is_error": True, + "result_chars": 99, + "duration_ms": 42, + } + ], + final_text="x", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + usage_scope="run", + ) + d = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=lambda t, o, a: "optimal", + ) + assert d["calls"][0]["is_error"] is True + assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) + assert d["calls"][0]["result_tokens_estimated"] is True + assert d["result_tokens_estimated"] is True + assert d["calls"][0]["duration_ms"] == 42 + assert d["errored_calls"] == 1 + + test_agent_run_dict_keeps_action_arg() + test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks() + test_agent_run_hit_max_maps_to_hit_max_iterations() + test_agent_run_to_harness_dict_does_not_guess_usage_total() + test_agent_run_to_harness_propagates_proxy_fields() def test_task_result_schema_round_trip_owns_usage_shape(): diff --git a/tests/evals/test_token_counting.py b/tests/evals/test_token_counting.py index a88ef560..650745b1 100644 --- a/tests/evals/test_token_counting.py +++ b/tests/evals/test_token_counting.py @@ -4,83 +4,90 @@ import sys +import pytest + from evals.results import AgentRun, agent_run_to_harness_dict from evals.runner.live import classify_call from evals.token_counting import estimate_result_tokens -def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): - class FakeEncoding: - def encode(self, text): - assert text == "serialized workspace result" - return [10, 20, 30] +def test_agent_behaviours(monkeypatch): + def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): + class FakeEncoding: + def encode(self, text): + assert text == "serialized workspace result" + return [10, 20, 30] - class FakeTiktoken: - @staticmethod - def get_encoding(name): - assert name == "cl100k_base" - return FakeEncoding() + class FakeTiktoken: + @staticmethod + def get_encoding(name): + assert name == "cl100k_base" + return FakeEncoding() - monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len("serialized workspace result"), - "result_text": "serialized workspace result", - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", - ) + monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len("serialized workspace result"), + "result_text": "serialized workspace result", + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, - ) + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) - assert out["calls"][0]["result_tokens"] == 3 - assert out["calls"][0]["result_tokens_estimated"] is False - assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" - assert out["result_tokens_estimated"] is False - assert out["result_tokens_mode"] == "measured" - assert "result_text" not in out["calls"][0] + assert out["calls"][0]["result_tokens"] == 3 + assert out["calls"][0]["result_tokens_estimated"] is False + assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" + assert out["result_tokens_estimated"] is False + assert out["result_tokens_mode"] == "measured" + assert "result_text" not in out["calls"][0] + def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): + monkeypatch.setitem(sys.modules, "tiktoken", None) + text = "payload without a tokenizer" + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len(text), + "result_text": text, + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) -def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): - monkeypatch.setitem(sys.modules, "tiktoken", None) - text = "payload without a tokenizer" - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len(text), - "result_text": text, - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", - ) + out = agent_run_to_harness_dict( + run, + optimal={"find_work_items"}, + alternate=set(), + classify=classify_call, + ) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, - ) + assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True - assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True + with pytest.MonkeyPatch.context() as mp: + test_agent_run_payload_uses_importable_tokenizer(mp) + with pytest.MonkeyPatch.context() as mp: + test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(mp) From acd6e3d4f7676b2810adc629d217ab886adf0588 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 14 Aug 2026 17:30:23 +0530 Subject: [PATCH 31/93] Tighten the long docstrings without dropping what they know Docstrings over six lines: 23 -> 9. The nine that remain are numbered rule sets or schema field lists, where compressing removes the thing you would read them for. Nothing was deleted for being long. The Claude usage envelope loses its pretty-printed JSON but keeps every field name and the trap that bare input_tokens is uncached-only, reading 8-10 while cache_read was 180k+. The cycle seeder keeps the constraint that makes W6 seed an open cycle. The antigravity fake home keeps "real directory, copied token, never symlinks", because a symlink means a token refresh writes into the real home. Co-Authored-By: Claude Opus 5 (1M context) --- evals/cleanup.py | 11 ++------ evals/drivers/api/backend.py | 12 ++------ evals/drivers/cli/antigravity.py | 32 +++++++--------------- evals/drivers/cli/claude.py | 33 ++++------------------ evals/drivers/cli/codex.py | 14 ++++------ evals/drivers/cli/opencode.py | 9 ++---- evals/drivers/cli/process.py | 25 +++++------------ evals/drivers/cli/sidecar.py | 11 +++----- evals/listing.py | 14 +++------- evals/proxy.py | 20 ++++---------- evals/results.py | 19 ++++--------- evals/seed/build.py | 12 +++----- evals/seed/cycles.py | 21 +++----------- evals/seed/projects.py | 47 ++++++++------------------------ evals/tasks/catalog.py | 18 ++++-------- evals/tasks/prompts.py | 12 +++----- evals/tasks/write.py | 11 +++----- 17 files changed, 89 insertions(+), 232 deletions(-) diff --git a/evals/cleanup.py b/evals/cleanup.py index 8f6f4246..b2d91e1b 100644 --- a/evals/cleanup.py +++ b/evals/cleanup.py @@ -1,12 +1,7 @@ -"""Delete leftover projects whose names start with a space-delimited prefix (default ``"EVAL "``). +"""Delete leftover projects whose names start with a prefix (default ``"EVAL "``). -Usage: - python -m evals.cleanup # dry-run: list only - python -m evals.cleanup --prefix "EVAL " # custom name prefix (note trailing space) - python -m evals.cleanup --yes # actually delete - -Uses EVAL_PLANE_API_KEY / EVAL_PLANE_WORKSPACE_SLUG via make_plane_client. -Default is dry-run; ``--yes`` is required to call delete. +``python -m evals.cleanup [--prefix "EVAL " | --yes]`` — dry-run lists only; ``--yes`` is +required before anything is deleted. Credentials come from EVAL_PLANE_* via make_plane_client. """ from __future__ import annotations diff --git a/evals/drivers/api/backend.py b/evals/drivers/api/backend.py index 22030830..25731e8c 100644 --- a/evals/drivers/api/backend.py +++ b/evals/drivers/api/backend.py @@ -19,15 +19,9 @@ class UnmappedModelTierError(ValueError): class StopReason(str, Enum): """Harness-owned reasons why a provider turn stopped. - Values intentionally preserve the strings historically emitted by the - Anthropic API path so old and new result rows remain comparable. Provider - adapters retain the provider's original value separately on ``Turn``. - - ``END_TURN`` is a normal completed response; ``TOOL_USE`` requests tool - execution; ``MAX_TOKENS`` and ``MODEL_CONTEXT_WINDOW_EXCEEDED`` are token - limits; ``REFUSAL`` is terminal and prevents requested side effects; - ``PAUSE_TURN`` asks the loop to continue; and ``UNKNOWN`` is the explicit - fallback for missing or newly introduced provider values. + Values keep the strings the Anthropic path historically emitted so old and new rows + stay comparable; adapters keep the provider's own value on ``Turn``. REFUSAL is + terminal and prevents side effects; UNKNOWN is the explicit fallback for new values. """ END_TURN = "end_turn" diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index 7530c387..e6ace9ec 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -44,18 +44,12 @@ def prepare_antigravity_fake_home( env: dict[str, str], real_home: Path | None = None, ) -> None: - """Build an isolated HOME for agy with MCP config + shared auth artifacts. - - Writes mcp_config.json to BOTH documented locations (cheap; live probe - should settle which path agy actually reads): - - ~/.gemini/config/mcp_config.json - - ~/.gemini/antigravity-cli/mcp_config.json - - Creates ``antigravity-cli`` as a **real directory** (never a symlink of the - whole tree — that would write mcp_config and runtime logs into real user - state). Auth artifacts (``antigravity-oauth-token``) are plain **copies** — - never symlinks — so an in-place token refresh cannot write through into the - real home. Staleness over a single eval run is negligible. + """Build an isolated HOME for agy with MCP config plus copied auth artifacts. + + Writes mcp_config.json to both documented paths (~/.gemini/config/ and + ~/.gemini/antigravity-cli/) since which one agy reads is unsettled. antigravity-cli is + a real directory and the oauth token a plain copy, never symlinks — otherwise a token + refresh or a runtime log would write through into the user's real home. """ real_home = real_home or Path.home() gemini_root = fake_home / ".gemini" @@ -97,16 +91,10 @@ def prepare_antigravity_fake_home( class AntigravityCliDriver(CliDriver): """Run tasks via Google Antigravity CLI (``agy``). - Probed flags (2026-08-12, ``agy --help``): - - ``-p`` / ``--print`` headless single-prompt mode - - ``--output-format`` text|json|stream-json - - ``--model``, ``--dangerously-skip-permissions`` - - MCP via ``~/.gemini/config/mcp_config.json`` (``mcpServers`` map; - stdio: command/args/env). No CLI flag for MCP config → HOME isolation. - - No max-turns / turn-cap flag in help → ``hit_max_turns=False`` + note. - - Tool calls come from the recording proxy sidecar (protocol-layer), not - agy stdout parsing. + Probed 2026-08-12: -p headless, --output-format text|json|stream-json, --model, + --dangerously-skip-permissions. MCP only via ~/.gemini/config/mcp_config.json with no + CLI flag, hence HOME isolation; no turn-cap flag, so hit_max_turns=False plus a note. + Tool calls come from the proxy sidecar, not from parsing agy stdout. """ name = "antigravity-cli" diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index 1c415583..f6c1c54e 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -22,34 +22,11 @@ def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: """Parse Claude print-mode usage into (raw_usage, usage_total). - Real envelope (probed 2026-08-12, ``claude -p --output-format json``):: - - { - "usage": { - "input_tokens": 10, # uncached NEW input only — NOT run total - "cache_creation_input_tokens": 17459, - "cache_read_input_tokens": 18464, - "output_tokens": 143, - "iterations": [...], - ... - }, - "modelUsage": { - "": { - "inputTokens": 10, - "outputTokens": 143, - "cacheReadInputTokens": 18464, - "cacheCreationInputTokens": 17459, - "costUSD": 0.037, - ... - } - }, - "total_cost_usd": 0.037 - } - - ``usage.input_tokens`` alone is misleading for multi-turn cached runs (live - rows showed 8–10 while cache_read was 180k+). We keep the split fields and - compute an inclusive total under ``usage_total``; callers must **not** copy - bare ``input_tokens`` into ``cum_input_tokens``. + The envelope splits input across ``input_tokens`` (uncached new input only), + ``cache_creation_input_tokens`` and ``cache_read_input_tokens``, mirrored in + ``modelUsage.`` as camelCase plus ``costUSD``/``total_cost_usd``. + Bare ``input_tokens`` is the trap: live multi-turn rows read 8-10 while + cache_read was 180k+, so callers must never copy it into ``cum_input_tokens``. """ usage = data.get("usage") if usage is not None and not isinstance(usage, dict): diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index d132f1a2..4858b596 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -162,16 +162,12 @@ def parse_codex_rollout_calls(rollout_path: Path) -> list[dict[str, Any]]: def find_codex_rollout(session_id: str | None, *, after_ts: float | None = None) -> Path | None: - """Find a rollout JSONL under ``~/.codex/sessions`` matching *session_id* exactly. + """Find the rollout JSONL under ~/.codex/sessions matching *session_id* exactly. - Matches filename containing the id (rollout filenames end with ``thread_id``) - or a first-line ``session_meta`` / ``thread.started`` id field. - - **No newest-after-ts fallback**: under parallel runs that would pick another - task's rollout and corrupt final_text. Callers should note - ``codex_rollout_unmatched`` when this returns None. - - ``after_ts`` is accepted for API compatibility but ignored. + Matches the filename (which ends with thread_id) or a first-line session_meta id. + Deliberately no newest-after-ts fallback: under parallel runs that picks another + task's rollout and corrupts final_text. Callers note codex_rollout_unmatched on None. + ``after_ts`` is accepted for API compatibility and ignored. """ del after_ts # intentionally unused — see docstring if not session_id: diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py index 04c66aa5..2816a26b 100644 --- a/evals/drivers/cli/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -43,12 +43,9 @@ def write_opencode_mcp_config( class OpencodeCliDriver(CliDriver): """Run tasks via ``opencode run`` (proxy-first call recording). - Probed flags (2026-08-12): - - ``opencode run [message..]`` non-interactive - - ``--format json|default``, ``-m/--model`` - - MCP via ``opencode.json`` ``mcp`` section (local: type/command/environment) - written into the task cwd (or a temp project dir). - - No turn-cap flag → ``hit_max_turns=False`` + note ``no_turn_cap``. + Probed 2026-08-12: ``opencode run [message..]`` non-interactive, --format json|default, + -m/--model. MCP comes from an ``opencode.json`` mcp section written into the task cwd; + no turn-cap flag, so hit_max_turns=False plus a ``no_turn_cap`` note. """ name = "opencode-cli" diff --git a/evals/drivers/cli/process.py b/evals/drivers/cli/process.py index f092cfe1..4299550b 100644 --- a/evals/drivers/cli/process.py +++ b/evals/drivers/cli/process.py @@ -15,13 +15,9 @@ def kill_process_group(proc: subprocess.Popen[Any]) -> bool: """SIGKILL the process group whose leader is ``proc``. - With ``start_new_session=True``, ``pgid == proc.pid`` even after the leader - has been reaped — call ``killpg(proc.pid, …)`` directly (never fall back to - killing only the leader, which leaves grandchildren alive). - - Returns True if ``killpg`` delivered the signal; False if the group is - already fully gone (``ProcessLookupError`` = success for cleanup, but the - kill itself did not run). + With start_new_session, pgid == proc.pid even after the leader is reaped, so killpg on + that pid directly — killing only the leader leaves grandchildren alive. Returns True if + the signal was delivered, False if the group was already gone. """ if proc.pid is None: return False @@ -71,17 +67,10 @@ def run_cli_subprocess( ) -> subprocess.CompletedProcess[Any]: """Run a CLI in its own process group; kill the **whole group** on timeout/interrupt. - Node wrappers (e.g. ``codex``) spawn native grandchildren. Plain - ``subprocess.run`` on timeout only kills the parent; the grandchild keeps - stdout open and ``communicate()`` hangs indefinitely. This runner: - - 1. launches with ``start_new_session=True`` (new process group; pgid=pid); - 2. on timeout **or any other exception** (incl. KeyboardInterrupt), - ``os.killpg(pid, SIGKILL)`` the group; - 3. drains pipes with a **bounded** second ``communicate`` (never unbounded). - - Raises ``subprocess.TimeoutExpired`` with attribute - ``killed_process_group=True`` only when killpg actually delivered the signal. + Node wrappers like ``codex`` spawn native grandchildren, and plain subprocess.run kills + only the parent — the grandchild holds stdout open and communicate() hangs forever. So: + start_new_session, killpg on any exception, then a bounded second communicate to drain. + Raises TimeoutExpired with ``killed_process_group=True`` only when the signal landed. """ popen_kwargs: dict[str, Any] = { "cwd": cwd, diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index e3a862f2..8786b518 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -181,14 +181,11 @@ def wait_for_proxy_meta( poll_s: float = 0.2, max_wait_s: float | None = None, ) -> bool: - """Poll until the sidecar gains a ``proxy_meta`` row (or the wait expires). + """Poll until the sidecar gains a ``proxy_meta`` row, returning True if it appears. - After a CLI timeout the driver kills the CLI; the proxy is a *separate* - process that only then sees stdin EOF and needs up to - ``SHUTDOWN_DEADLINE_S`` to flush call rows + meta. Call this **before** - harvesting so the temp dir is not deleted mid-finalization. - - Returns True if meta was observed. + The proxy is a separate process: after the driver kills the CLI it only then sees stdin + EOF and needs up to SHUTDOWN_DEADLINE_S to flush. Call before harvesting so the temp + directory is not deleted mid-finalization. """ # Local import keeps drivers import-light for non-proxy unit tests. from evals.proxy import SHUTDOWN_DEADLINE_S diff --git a/evals/listing.py b/evals/listing.py index 4fedc1de..d1a64ba9 100644 --- a/evals/listing.py +++ b/evals/listing.py @@ -1,14 +1,8 @@ -"""Measure MCP tool listing size (tool count + cl100k tokens). +"""Measure MCP tool listing size: tool count and cl100k tokens. -Usage: - python -m evals.listing --label local - python -m evals.listing --server-cmd '/path/bin/python -m plane_mcp stdio' --server-env KEY=VAL - -Reports: tool count, wire listing tokens (incl. outputSchema), model-facing tokens -(minus outputSchema), and the top-10 tools by wire tokens. - -Credentials: EVAL_PLANE_API_KEY, EVAL_PLANE_WORKSPACE_SLUG, optional EVAL_PLANE_BASE_URL. -tiktoken is a **dev** optional dependency (same as scripts/check_token_budget.py). +``python -m evals.listing [--label local | --server-cmd '' --server-env KEY=VAL]`` +Reports wire tokens (with outputSchema), model-facing tokens (without), and the top-10 +tools by size. Needs EVAL_PLANE_* credentials; tiktoken is a dev optional dependency. """ from __future__ import annotations diff --git a/evals/proxy.py b/evals/proxy.py index 22c9b9c4..7cfce9fb 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -1,19 +1,9 @@ -"""Stdio MCP recording proxy — byte-faithful JSON-RPC relay with sidecar call log. +"""Stdio MCP recording proxy — byte-faithful JSON-RPC relay with a sidecar call log. -Usage: - python -m evals.proxy --log SIDECAR.jsonl [--record-result-payloads] -- - -Spawns the target as a child, relays parent stdin → child stdin and child -stdout → parent stdout as raw bytes (byte-faithful; does not re-serialize). -Parses complete newline-delimited JSON lines from a *copy* of each direction -to record ``tools/call`` request/response pairs into the sidecar JSONL. - -I/O uses ``os.read`` on raw fds + per-direction bytearray buffers — never -``select`` + buffered ``readline`` (partial lines would hang; buffered -prefetch would stall multi-line clients). - -Child stderr is forwarded to our stderr. Exit code matches the child -(negative/signal codes map to conventional 128+signum). +``python -m evals.proxy --log SIDECAR.jsonl [--record-result-payloads] -- `` +Relays raw bytes both ways and parses a *copy* to log tools/call pairs. Uses ``os.read`` on +raw fds, never select + buffered readline: partial lines hang and prefetch stalls multi-line +clients. Child stderr is forwarded; exit code matches the child (signals as 128+signum). """ from __future__ import annotations diff --git a/evals/results.py b/evals/results.py index 33ae7e1c..97bfbd0a 100644 --- a/evals/results.py +++ b/evals/results.py @@ -87,12 +87,9 @@ class AgentRun: class TaskResult: """One task repetition and the complete persisted row schema. - ``schema_version=0`` identifies rows written before this type existed; - :meth:`from_row` supplies defaults for every field added since. Version 1 - defines ``wall_time_s`` as CLI invocation time only, excluding harness-owned - config/temp-directory setup. Pre-versioned Claude, Antigravity, and OpenCode - rows include a few milliseconds of that setup; Codex and API rows already - used invocation/agent-loop timing. + schema_version 0 marks rows written before this type existed; from_row defaults every + field added since. Version 1 defines wall_time_s as CLI invocation time only — earlier + Claude/Antigravity/OpenCode rows also include a few ms of harness setup. """ schema_version: int = RESULT_SCHEMA_VERSION @@ -446,13 +443,9 @@ def agent_run_to_task_result( ) -> TaskResult: """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. - Only **plane** MCP tools are classified and counted in ``num_calls`` / - mispick metrics. Client built-ins (``ToolSearch``, …) go to - ``client_tool_calls`` and are excluded. - - CLI drivers never populate ``cum_input_tokens`` from bare - ``usage.input_tokens`` (that field is uncached-only under Claude Code and - misreads multi-turn cached runs as ~10 tokens). Use ``usage_total`` instead. + Only Plane MCP tools count toward num_calls and mispicks; client built-ins go to + client_tool_calls. CLI drivers never fill cum_input_tokens from bare usage.input_tokens + — under Claude Code that is uncached-only and misreads cached runs as ~10 tokens. """ # Re-split in case callers passed a mixed list plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) diff --git a/evals/seed/build.py b/evals/seed/build.py index e59db02a..9140253e 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -31,15 +31,11 @@ def remove_stale_workspace_artifacts(plane: PlaneClient, workspace_slug: str) -> None: - """Delete leftover WS3 long-tail artifacts so a dirty workspace cannot false-pass. + """Delete leftover release tag ``eval-rc1`` and customer property ``Eval Industry``. - Removes any existing release tag ``eval-rc1`` and customer property - ``Eval Industry`` before the rep seeds. - - Empty / not-found lists are silent. Clients without the API surface (offline - test stubs) are skipped silently. If a matching artifact is **found** and - cannot be deleted — or list fails on a present API — raises so the harness - records ``infra_seed`` rather than running against dirty state. + A dirty workspace would otherwise false-pass. Missing lists and clients without the + API surface are silent, but a found-and-undeletable artifact raises so the harness + records infra_seed instead of grading against dirty state. """ releases = getattr(plane, "releases", None) tags_api = getattr(releases, "tags", None) if releases is not None else None diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py index 65841279..7b4bf4ef 100644 --- a/evals/seed/cycles.py +++ b/evals/seed/cycles.py @@ -23,23 +23,10 @@ def seed_cycles( ) -> None: """Seed Sprint 12 (past) + Sprint 13 (active) with work items. - Plane forbids adding issues to a cycle whose end_date is already past - (``The Cycle has already been completed so no new issues can be added`` — - plane-ee cycle/issue.py). Ordering for Sprint 12: - - 1. create with an *active* window (start past, end future) - 2. add_work_items while still active - 3. update end_date to the past (backdate) so the cycle is completed - - ``leave_past_open`` skips step 3, leaving Sprint 12 ending tomorrow. Closing a - cycle is only legal while it is still open — Plane rejects every edit to an - ended cycle (``The Cycle has already been completed so it cannot be edited``) - and rejects a transfer out of a still-running one (``The old cycle is not - completed yet``), so a fixture that pre-closes Sprint 12 makes "close it" - unachievable and leaves ``progress_snapshot`` (a transfer side effect) as the - only observable close signal. W6 asks the agent to close, so it seeds open. - - Sprint 13 is created and populated while genuinely active (start ≤ today ≤ end). + Plane refuses to add issues to an ended cycle, so Sprint 12 is created with an active + window, filled, then backdated. ``leave_past_open`` skips the backdate: Plane also + rejects every edit to an ended cycle, so pre-closing it makes W6's "close it" + unachievable and leaves progress_snapshot (a transfer side effect) as the only signal. """ project_id = context["project_id"] me_id = context.get("me_id") or str(plane.users.get_me().id) diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 0c2da6d2..74e930bd 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -32,17 +32,12 @@ def is_plan_gate(exc: BaseException) -> bool: - """True only for genuine plan/subscription feature gates — not generic API failures. + """True only for genuine plan gates — not generic API failures. - 402 is unambiguous: ``check_feature_flag`` returns it for a plan refusal and nothing - else in this API uses it. - - 403 and 400 are not. Plane raises 403 for an ordinary permission denial *and* for the - plan gates on initiatives, teamspaces and some workflow routes, in the same - ``{"detail": ...}`` shape; 400 covers every serializer validation error as well as a - few plan refusals. Treating a bare 403 as a gate meant a genuine permission failure - was recorded as an environment skip — the harness quietly hiding the class of defect - it exists to find. Those two statuses now need the refusal to say so. + 402 is unambiguous. 403 and 400 are not: Plane uses 403 for ordinary permission denial + and for the initiative/teamspace plan gates in the same shape, so a bare 403 counted as + a gate turned real permission bugs into environment skips. Those two now need the + refusal to name a plan limit. """ if not isinstance(exc, HttpError): return False @@ -58,15 +53,9 @@ def is_plan_gate(exc: BaseException) -> bool: def plan_gate_skips(feature: str) -> Iterator[None]: """Turn a plan refusal raised inside the block into a task skip. - ``DESIGN.md`` states that a plan gate is not rewritten as an agent task failure, but - an uncaught seed exception becomes ``infra_seed`` and the whole task-rep dies. A - capability the workspace's plan does not include is an environment fact, so it is - recorded the way L2 records its missing activity worker: a skip carrying a reason, - excluded from success denominators. - - ``TaskSkipped`` is imported here rather than at module scope because - ``evals.tasks.skip`` cannot be reached without initialising ``evals.tasks``, whose - task modules import this package. + An uncaught seed exception becomes infra_seed and kills the task-rep; a capability the + plan excludes is an environment fact, recorded like L2's missing activity worker. + TaskSkipped is imported inside because evals.tasks imports this package at module load. """ from evals.tasks.skip import TaskSkipped @@ -157,23 +146,11 @@ def enable_workspace_features( *, exclude: set[str] | frozenset[str] | None = None, ) -> dict[str, bool | None]: - """Set workspace-level feature toggles to what task preconditions need. - - Gate (plane-ee): create-customer 403 when - ``check_workspace_feature(slug, IS_CUSTOMER_ENABLED)`` is false — DB column - ``WorkspaceFeature.is_customer_enabled``. Legacy/SDK flips it via - ``workspaces.update_features`` / ``WorkspaceFeature(customers=True)`` - (API serializer maps ``customers`` → ``is_customer_enabled``). - - Deliberately does **not** set ``work_item_types``: that flips - workspace-vs-project type ownership and would change S1/S3 seed mode. - - An excluded feature is written as ``False``, not left alone. A workspace outlives - every run, so omitting the write leaves whatever the last task-rep put there — - which silently satisfied S5's customers precondition on every run after the first. + """Set workspace-level feature toggles, returning the prior values for teardown. - Returns the prior values so teardown can restore them; the harness runs against a - Plane instance it does not own and should not leave configuration drift behind. + Excluded features are written ``False``, not skipped: the workspace outlives every run, + so omitting the write silently satisfied S5's customers precondition after run one. + Never sets ``work_item_types`` — that flips type ownership and changes S1/S3 seed mode. """ skip = set(exclude or ()) prior = workspace_feature_state(plane, workspace_slug) diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 3b5a812f..73496ae4 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -96,18 +96,12 @@ def task_author(task: dict[str, Any]) -> str: def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: - """Stable short hash of the task battery used for a run. - - SHA-256 (first 12 hex chars) over a canonical serialization of ``CATALOG_REVISION`` - and every task sorted by id: id, prompt, sorted optimal/alternate tools, and - optimal_calls. - - Ceilings (intentionally *not* covered per-task by the hash): - - Verifier functions and ``needs`` fixtures do not alter the fingerprint on their - own — prompt/tool-set drift is the stability signal. Bump ``CATALOG_REVISION`` - when they change in a way that redefines the question. - - The hash covers the *selected* task list: ``--tasks`` subsets produce - different fingerprints than a full-catalog run. + """Stable short hash (SHA-256/12) of CATALOG_REVISION plus each task's id, prompt, + tool sets and optimal_calls. + + Fixtures and verifier bodies are deliberately excluded — prompt drift is the signal — + so bump CATALOG_REVISION when they redefine the question. A --tasks subset hashes + differently from the full catalog. """ src = list(TASKS if tasks is None else tasks) payload: list[dict[str, Any]] = [] diff --git a/evals/tasks/prompts.py b/evals/tasks/prompts.py index 88be89eb..8d2448a3 100644 --- a/evals/tasks/prompts.py +++ b/evals/tasks/prompts.py @@ -18,14 +18,10 @@ def format_task_prompt( ) -> str: """Render a task prompt with seed-bound placeholders. - Always provides ``project`` (from ctx or a dry-run sample). Tasks that hand - the agent concrete UUIDs / PROJ-N identifiers supply extra keys via an - optional ``prompt_bind(ctx) -> dict`` callable on the task dict. - - When ``strict=True`` (live runs), empty-string values or binder exceptions - raise ``PromptBindError`` so the harness records ``infra_seed`` rather than - sending a blank-ID prompt to the agent. Dry-run uses ``strict=False`` and - fills missing keys with explicit ```` markers. + Always supplies ``project``; tasks needing concrete UUIDs add keys via an optional + ``prompt_bind(ctx)``. Live runs use strict=True so an empty value or a binder error + raises PromptBindError and is recorded infra_seed, rather than sending the agent a + blank ID; dry runs fill missing keys with ```` markers instead. """ tpl = str(task.get("prompt") or "") fields: dict[str, Any] = { diff --git a/evals/tasks/write.py b/evals/tasks/write.py index 379a13d8..16558778 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -549,13 +549,10 @@ def _time_tracking_enabled(plane: Any, workspace_slug: str, project_id: str) -> async def verify_w11(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: """W11: the work log lands even though the project shipped with time tracking off. - Same end state as W8, reached from an obstacle: the worklog endpoint 404s with - "Worklog is not enabled for the project" until the agent turns the feature on. The - prompt authorises that, so enabling and proceeding is the expected route rather than - an overreach the verifier would be rewarding. - - The failure notes separate the ways it can go wrong, because "no work log" alone does - not say whether the agent gave up, half-finished, or claimed success it never earned. + W8's end state reached from an obstacle — the worklog endpoint 404s until the feature + is on, which the prompt authorises. Failure notes separate the routes to failure, + since "no work log" alone does not say whether the agent gave up, half-finished, or + claimed a success it never earned. """ workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] From ef8228170d3356f349a2124bd9e1e03251449e69 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 18:32:17 +0530 Subject: [PATCH 32/93] Make fixture seeding and teardown own what they touch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teardown could delete workspace data the harness never created: any customer named "Acme Corp", every type-scoped Severity property, any workspace Incident type. Tracking-only deletion is not sufficient because some fixtures are created by the agent (C1's customer, S3's Incident) and never enter workspace_objects. Seeding now snapshots the ids of fixture-named workspace objects before the agent runs, and teardown deletes only tracked ids plus name matches absent from that baseline. An unreadable baseline disables name-based deletion rather than falling back to it — leaking a fixture is recoverable, deleting a stranger's data is not. The pre-seed purge that deleted leftover sentinels is now a detector: a pre-existing sentinel raises env:fixture-collision rather than being destroyed, scoped to the fixture categories each task actually uses so one orphan cannot skip the whole battery. cleanup.py gained --sentinels so the remediation the skip message names can actually be performed. Seeded truth is randomised per run, so read answers cannot be guessed from constants in this source tree, and a prompt-derived test fails CI if a task references a sentinel without collision coverage. Co-Authored-By: Claude Opus 5 (1M context) --- evals/changelog.py | 41 + evals/cleanup.py | 141 +- evals/fixtures.py | 128 + evals/seed/__init__.py | 25 +- evals/seed/build.py | 293 ++- evals/seed/customers.py | 17 +- evals/seed/cycles.py | 103 +- evals/seed/intake.py | 3 +- evals/seed/item_types.py | 65 +- evals/seed/modules.py | 9 +- evals/seed/plan.py | 17 +- evals/seed/projects.py | 110 +- evals/seed/randomize.py | 44 + evals/seed/releases.py | 72 +- evals/seed/remove.py | 365 ++- evals/seed/states.py | 57 + evals/seed/work_items.py | 400 +++- evals/seed/workspace.py | 22 + tests/evals/seed/test_gate_tolerance.py | 129 ++ tests/evals/seed/test_plan_gate.py | 2 +- tests/evals/seed/test_read_randomization.py | 240 ++ tests/evals/seed/test_seed.py | 2056 ++++++++++++----- ..._rows.jsonl => evals_schema_v0_rows.jsonl} | 0 23 files changed, 3445 insertions(+), 894 deletions(-) create mode 100644 evals/changelog.py create mode 100644 evals/fixtures.py create mode 100644 evals/seed/randomize.py create mode 100644 evals/seed/states.py create mode 100644 evals/seed/workspace.py create mode 100644 tests/evals/seed/test_read_randomization.py rename tests/fixtures/{evals_historical_rows.jsonl => evals_schema_v0_rows.jsonl} (100%) diff --git a/evals/changelog.py b/evals/changelog.py new file mode 100644 index 00000000..220cc3fd --- /dev/null +++ b/evals/changelog.py @@ -0,0 +1,41 @@ +"""Shared release changelog response normalization.""" + +from __future__ import annotations + +import re +from html import unescape +from typing import Any + + +def _field(value: Any, name: str) -> Any: + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + +def normalize_changelog_text(value: Any) -> str: + """Extract normalized text from a changelog API response or stored text.""" + nested = _field(value, "changelog") + candidates = ( + value if isinstance(value, str) else _field(value, "description_html"), + nested if isinstance(nested, str) else _field(nested, "description_html"), + ) + for candidate in candidates: + if isinstance(candidate, str) and candidate.strip(): + without_tags = re.sub(r"<[^>]*>", " ", candidate) + return " ".join(unescape(without_tags).split()) + return "" + + +def changelog_items(value: Any) -> list[str]: + """Extract exact item text following each ``Changelog entry …:`` label.""" + text = normalize_changelog_text(value) + markers = list(re.finditer(r"Changelog entry\s+[^:]+:\s*", text, flags=re.IGNORECASE)) + items: list[str] = [] + for index, marker in enumerate(markers): + end = markers[index + 1].start() if index + 1 < len(markers) else len(text) + item = text[marker.end() : end].strip().rstrip(".").strip() + if item: + items.append(item) + return items + + +__all__ = ["changelog_items", "normalize_changelog_text"] diff --git a/evals/cleanup.py b/evals/cleanup.py index b2d91e1b..e079fc64 100644 --- a/evals/cleanup.py +++ b/evals/cleanup.py @@ -1,7 +1,7 @@ -"""Delete leftover projects whose names start with a prefix (default ``"EVAL "``). +"""Delete leftover eval projects or fixed-name workspace sentinels. -``python -m evals.cleanup [--prefix "EVAL " | --yes]`` — dry-run lists only; ``--yes`` is -required before anything is deleted. Credentials come from EVAL_PLANE_* via make_plane_client. +``python -m evals.cleanup [--prefix "EVAL " | --sentinels] [--yes]`` — dry-run lists only; +``--yes`` is required before anything is deleted. Credentials come from EVAL_PLANE_*. """ from __future__ import annotations @@ -12,6 +12,21 @@ from plane.models.query_params import PaginatedQueryParams +from evals.seed.customers import ( + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, +) +from evals.seed.item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, +) +from evals.seed.releases import EVALUATION_RELEASE_TAG_VERSION +from evals.seed.workspace import list_workspace_rows + def list_projects_with_prefix(plane: Any, workspace_slug: str, prefix: str) -> list[Any]: """Return projects whose name starts with ``prefix`` (paginated list). @@ -61,13 +76,126 @@ def delete_projects( return deleted, failed +def list_sentinel_workspace_artifacts(plane: Any, workspace_slug: str) -> list[dict[str, Any]]: + """Return fixed-name workspace fixtures that can false-pass eval tasks.""" + customers = plane.customers + specs = ( + ( + "customer", + customers, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + lambda row: (getattr(row, "name", None) or "").strip(), + ), + ( + "release_tag", + plane.releases.tags, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + lambda row: (getattr(row, "version", None) or "").strip(), + ), + ( + "customer_property", + customers.properties, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + lambda row: (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip(), + ), + ) + artifacts: list[dict[str, Any]] = [] + for kind, api, matches, display_name in specs: + for row in list_workspace_rows(api, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is not None and matches(row): + artifacts.append({"kind": kind, "id": object_id, "name": display_name(row)}) + + type_api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(type_api, "list", None)): + for row in list_workspace_work_item_types(plane, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is not None and is_work_item_type_named(row, INCIDENT_TYPE_NAME): + artifacts.append({"kind": "work_item_type", "id": object_id, "name": INCIDENT_TYPE_NAME}) + + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if callable(getattr(property_api, "list", None)) and callable(getattr(links_api, "list", None)): + for row in list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME): + object_id = getattr(row, "id", None) + if object_id is not None and is_severity_property(row): + display = getattr(row, "display_name", None) or getattr(row, "name", None) or "" + artifacts.append({"kind": "work_item_property", "id": object_id, "name": display.strip()}) + return artifacts + + +def _sentinel_description(artifact: dict[str, Any]) -> str: + kind = str(artifact["kind"]).replace("_", " ") + return f"{kind} {artifact['name']!r} ({artifact['id']})" + + +def delete_sentinel_workspace_artifacts( + plane: Any, + workspace_slug: str, + artifacts: list[dict[str, Any]], + *, + yes: bool, +) -> tuple[int, int]: + """Delete explicitly selected sentinel artifacts. Returns (deleted, failed).""" + if not yes: + return 0, 0 + deleted = failed = 0 + for artifact in artifacts: + try: + if artifact["kind"] == "customer": + plane.customers.delete(workspace_slug=workspace_slug, customer_id=artifact["id"]) + elif artifact["kind"] == "release_tag": + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=artifact["id"]) + elif artifact["kind"] == "customer_property": + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=artifact["id"]) + elif artifact["kind"] == "work_item_type": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=artifact["id"]) + elif artifact["kind"] == "work_item_property": + plane.workspace_work_item_properties.delete( + workspace_slug=workspace_slug, + property_id=artifact["id"], + ) + else: + raise ValueError(f"unknown sentinel kind: {artifact['kind']}") + deleted += 1 + print(f" deleted sentinel {_sentinel_description(artifact)}") + except Exception as exc: + failed += 1 + print(f" FAILED sentinel {_sentinel_description(artifact)}: {exc}", file=sys.stderr) + return deleted, failed + + +def _cleanup_sentinels(plane: Any, workspace_slug: str, *, yes: bool) -> int: + artifacts = list_sentinel_workspace_artifacts(plane, workspace_slug) + print(f"workspace={workspace_slug} sentinel_matches={len(artifacts)}") + if not artifacts: + print("nothing to delete") + return 0 + if not yes: + for artifact in artifacts: + print(f" would delete sentinel {_sentinel_description(artifact)}") + print("dry-run: re-run with --sentinels --yes to delete these sentinel fixture(s)") + return 0 + deleted, failed = delete_sentinel_workspace_artifacts(plane, workspace_slug, artifacts, yes=True) + print(f"summary: deleted={deleted} failed={failed} matched={len(artifacts)}") + return 1 if failed else 0 + + def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description="Clean up leftover EVAL projects (dry-run by default)") + p = argparse.ArgumentParser(description="Clean up leftover eval fixtures (dry-run by default)") p.add_argument("--prefix", type=str, default="EVAL ", help='Project name prefix (default: "EVAL ")') + p.add_argument( + "--sentinels", + action="store_true", + help="Clean fixed-name workspace sentinels instead of projects", + ) p.add_argument( "--yes", action="store_true", - help="Actually delete matched projects (default is dry-run list only)", + help="Actually delete matched objects (default is dry-run list only)", ) args = p.parse_args(argv) @@ -79,6 +207,9 @@ def main(argv: list[str] | None = None) -> int: print(f"error: {exc}", file=sys.stderr) return 2 + if args.sentinels: + return _cleanup_sentinels(plane, workspace_slug, yes=args.yes) + projects = list_projects_with_prefix(plane, workspace_slug, args.prefix) print(f"workspace={workspace_slug} prefix={args.prefix!r} matches={len(projects)}") for proj in projects: diff --git a/evals/fixtures.py b/evals/fixtures.py new file mode 100644 index 00000000..c78a6c67 --- /dev/null +++ b/evals/fixtures.py @@ -0,0 +1,128 @@ +"""Neutral fixture names shared by seeders, task prompts, and cleanup. + +This module must not import either :mod:`evals.seed` or :mod:`evals.tasks`; both +packages re-export these names for backward compatibility. +""" + +from __future__ import annotations + +CUSTOMER_NAME = "Acme Corp" +CUSTOMER_REQUEST_NAME = "SSO support" +EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" +_EVALUATION_CUSTOMER_NAMES = {CUSTOMER_NAME.casefold(), "acme"} + +RELEASE_NAME = "1.2.0" +RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +EVALUATION_RELEASE_TAG_VERSION = "eval-rc1" + +INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" +INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" + +CYCLE_PAST = "Sprint 12" +CYCLE_CURRENT = "Sprint 13" + +MODULE_NAME = "Checkout revamp" +MODULE_COMPLETED_TITLES = ( + "Module done: cart totals", + "Module done: tax lines", + "Module done: shipping quote", +) + +# Fixed fixture titles for the ``items`` group. Exactly four are urgent. +WORK_ITEM_FIXTURES: list[tuple[str, str]] = [ + ("Payment webhook drops retries", "urgent"), + ("Checkout times out on 3DS challenge", "urgent"), + ("Session cookie not rotated after login", "urgent"), + ("Inventory count goes negative under load", "urgent"), + ("Search results ignore archived projects", "high"), + ("CSV export truncates multi-byte chars", "high"), + ("Webhook secret rotation docs missing", "medium"), + ("Dark mode contrast fails WCAG AA", "medium"), + ("Onboarding email template stale", "medium"), + ("Sidebar collapse flickers on resize", "low"), + ("Tooltip clipped inside modal dialog", "low"), + ("Footer year still says 2024", "none"), +] + +PAYMENT_WEBHOOK_TITLE = WORK_ITEM_FIXTURES[0][0] +CHECKOUT_TIMEOUT_TITLE = "Checkout times out on 3DS challenge" +CHECKOUT_COMMENT_PHRASES = ( + "stripe callback race", + "retry budget exhausted", +) +SIDEBAR_TITLE = "Sidebar collapse flickers on resize" +DARK_MODE_TITLE = "Dark mode contrast fails WCAG AA" +BLOCKING_SOURCE_TITLE = "Search results ignore archived projects" +BLOCKING_TARGET_TITLE = "CSV export truncates multi-byte chars" +BLOCKING_REFERENCE_ADDRESS = "https://example.com/eval/runbook-w7" +DUE_THIS_WEEK_TITLES = ( + "Webhook secret rotation docs missing", + "Onboarding email template stale", +) +UNFINISHED_CYCLE_TITLES = ( + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", +) + +# Historical public aliases used by task catalog modules and downstream scripts. +DEBIAS_CUSTOMER_PROP_DISPLAY = EVALUATION_CUSTOMER_PROPERTY_NAME +DEBIAS_RELEASE_TAG_VERSION = EVALUATION_RELEASE_TAG_VERSION +ITEM_FIXTURES = WORK_ITEM_FIXTURES +R1_TITLE = PAYMENT_WEBHOOK_TITLE +R3_DUE_TITLES = DUE_THIS_WEEK_TITLES +R5_COMMENT_PHRASES = CHECKOUT_COMMENT_PHRASES +R5_TITLE = CHECKOUT_TIMEOUT_TITLE +W2_TITLE = SIDEBAR_TITLE +W3_TITLE = DARK_MODE_TITLE +W6_UNFINISHED_TITLES = UNFINISHED_CYCLE_TITLES +W7_SOURCE_TITLE = BLOCKING_SOURCE_TITLE +W7_TARGET_TITLE = BLOCKING_TARGET_TITLE +W7_URL = BLOCKING_REFERENCE_ADDRESS +W8_TITLE = PAYMENT_WEBHOOK_TITLE + + +def is_evaluation_customer_name(name: str | None) -> bool: + """Return whether a customer name matches an eval fixture alias.""" + return (name or "").strip().casefold() in _EVALUATION_CUSTOMER_NAMES + + +__all__ = [ + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "CYCLE_CURRENT", + "CYCLE_PAST", + "DARK_MODE_TITLE", + "DEBIAS_CUSTOMER_PROP_DISPLAY", + "DEBIAS_RELEASE_TAG_VERSION", + "DUE_THIS_WEEK_TITLES", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "EVALUATION_RELEASE_TAG_VERSION", + "INTAKE_BILLING_TITLE", + "INTAKE_SPAM_TITLE", + "ITEM_FIXTURES", + "MODULE_COMPLETED_TITLES", + "MODULE_NAME", + "PAYMENT_WEBHOOK_TITLE", + "R1_TITLE", + "R3_DUE_TITLES", + "R5_COMMENT_PHRASES", + "R5_TITLE", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "SIDEBAR_TITLE", + "UNFINISHED_CYCLE_TITLES", + "W2_TITLE", + "W3_TITLE", + "W6_UNFINISHED_TITLES", + "W7_SOURCE_TITLE", + "W7_TARGET_TITLE", + "W7_URL", + "W8_TITLE", + "WORK_ITEM_FIXTURES", + "is_evaluation_customer_name", +] diff --git a/evals/seed/__init__.py b/evals/seed/__init__.py index 93904abf..25307e34 100644 --- a/evals/seed/__init__.py +++ b/evals/seed/__init__.py @@ -1,12 +1,12 @@ """Evaluation fixture creation and removal.""" -from .build import remove_stale_workspace_artifacts, seed -from .build import remove_stale_workspace_artifacts as _preclean_ws3_workspace_artifacts +from .build import check_workspace_fixture_collisions, collision_categories, seed from .client import make_plane_client from .customers import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, seed_customer, ) from .customers import ( @@ -14,12 +14,18 @@ ) from .cycles import CYCLE_CURRENT, CYCLE_PAST, seed_cycles from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, seed_intake -from .item_types import seed_item_type +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + seed_item_type, +) from .labels import LABEL_NAMES, seed_labels from .modules import MODULE_COMPLETED_TITLES, MODULE_NAME, seed_module from .plan import seed_plan from .projects import ( MAIN_PROJECT_BUG_TITLES, + PLANE_PROJECT_IDENTIFIER_MAX_LENGTH, SECOND_PROJECT_BUG_TITLES, create_project_with_identifier_retry, enable_project_features, @@ -44,7 +50,7 @@ from .releases import ( EVALUATION_RELEASE_TAG_VERSION as DEBIAS_RELEASE_TAG_VERSION, ) -from .remove import teardown +from .remove import CleanupFailure, TeardownError, teardown from .work_items import ( BLOCKING_REFERENCE_ADDRESS, BLOCKING_SOURCE_TITLE, @@ -101,8 +107,10 @@ from .work_items import require_activities as _gate_activity_worker __all__ = [ + "BUG_TYPE_NAME", "CUSTOMER_NAME", "CUSTOMER_REQUEST_NAME", + "CleanupFailure", "CYCLE_CURRENT", "CYCLE_PAST", "DEBIAS_CUSTOMER_PROP_DISPLAY", @@ -114,11 +122,13 @@ "INTAKE_BILLING_TITLE", "INTAKE_SPAM_TITLE", "ITEM_FIXTURES", + "INCIDENT_TYPE_NAME", "LABEL_NAMES", "MAIN_PROJECT_BUG_TITLES", "MODULE_COMPLETED_TITLES", "MODULE_NAME", "PAYMENT_WEBHOOK_TITLE", + "PLANE_PROJECT_IDENTIFIER_MAX_LENGTH", "R1_TITLE", "R3_DUE_TITLES", "R5_COMMENT_PHRASES", @@ -128,7 +138,9 @@ "RELEASE_CHANGELOG_TEXT", "RELEASE_NAME", "SECOND_PROJECT_BUG_TITLES", + "SEVERITY_PROPERTY_NAME", "SIDEBAR_TITLE", + "TeardownError", "UNFINISHED_CYCLE_TITLES", "W2_TITLE", "W3_TITLE", @@ -144,16 +156,17 @@ "CHECKOUT_COMMENT_PHRASES", "CHECKOUT_TIMEOUT_TITLE", "_gate_activity_worker", - "_preclean_ws3_workspace_artifacts", + "check_workspace_fixture_collisions", + "collision_categories", "create_project_with_identifier_retry", "enable_project_features", "enable_workspace_features", "find_completed_state", "is_identifier_collision", + "is_evaluation_customer_name", "is_plan_gate", "list_states", "make_plane_client", - "remove_stale_workspace_artifacts", "require_activities", "secrets", "seed", diff --git a/evals/seed/build.py b/evals/seed/build.py index 9140253e..c2bfe2cc 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -7,10 +7,27 @@ from plane import PlaneClient -from .customers import EVALUATION_CUSTOMER_PROPERTY_NAME, seed_customer +from evals.errors import TaskSkipped + +from .customers import ( + CUSTOMER_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, + seed_customer, +) from .cycles import seed_cycles from .intake import seed_intake -from .item_types import seed_item_type +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, + seed_item_type, + workspace_owns_work_item_types, +) from .labels import seed_labels from .modules import seed_module from .projects import ( @@ -20,6 +37,7 @@ seed_second_project, ) from .releases import EVALUATION_RELEASE_TAG_VERSION, seed_release +from .states import seed_r7_state_oracle from .work_items import ( CHECKOUT_COMMENT_PHRASES, CHECKOUT_TIMEOUT_TITLE, @@ -28,66 +46,223 @@ require_activities, seed_work_items, ) +from .workspace import list_workspace_rows +_WORKSPACE_BASELINE_CATEGORIES = ( + "customers", + "release_tags", + "customer_properties", + "work_item_types", + "work_item_properties", +) +_TASK_COLLISION_CATEGORIES = { + "C1": {"customers"}, + "L3": {"release_tags"}, + "S1": {"work_item_properties"}, + "S3": {"work_item_types"}, +} -def remove_stale_workspace_artifacts(plane: PlaneClient, workspace_slug: str) -> None: - """Delete leftover release tag ``eval-rc1`` and customer property ``Eval Industry``. - A dirty workspace would otherwise false-pass. Missing lists and clients without the - API surface are silent, but a found-and-undeletable artifact raises so the harness - records infra_seed instead of grading against dirty state. - """ +def _snapshot_workspace_baseline( + plane: PlaneClient, + workspace_slug: str, + categories: set[str] | None = None, +) -> dict[str, set[str] | None]: + """Capture fixed-name workspace fixtures that existed before the agent runs.""" + baseline: dict[str, set[str] | None] = dict.fromkeys(_WORKSPACE_BASELINE_CATEGORIES) + # Preserve the established always-on snapshots. Type APIs may be plan-gated, so + # their ownership baselines are read only for tasks that can create those fixtures. + wanted = {"customers", "release_tags", "customer_properties"} | set(categories or ()) + customers = getattr(plane, "customers", None) releases = getattr(plane, "releases", None) - tags_api = getattr(releases, "tags", None) if releases is not None else None - if tags_api is not None: + specs = ( + ( + "customers", + customers, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + ), + ( + "release_tags", + getattr(releases, "tags", None) if releases is not None else None, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + ), + ( + "customer_properties", + getattr(customers, "properties", None) if customers is not None else None, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + ), + ) + for category, api, matches in specs: + if category not in wanted: + continue + if api is None: + continue try: - page = tags_api.list(workspace_slug=workspace_slug) + rows = list_workspace_rows(api, workspace_slug) except Exception as exc: - raise RuntimeError(f"WS3 preclean: list release tags failed: {exc}") from exc - rows = page.results if hasattr(page, "results") else page - for tag in rows or []: - version = (getattr(tag, "version", None) or "").strip() - if version != EVALUATION_RELEASE_TAG_VERSION: - continue - tag_id = getattr(tag, "id", None) - if not tag_id: - continue + raise RuntimeError(f"workspace baseline snapshot: list {category} failed: {exc}") from exc + baseline[category] = {str(row.id) for row in rows if getattr(row, "id", None) is not None and matches(row)} + + if "work_item_types" in wanted: + api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(api, "list", None)): try: - tags_api.delete(workspace_slug=workspace_slug, tag_id=tag_id) + rows = ( + list_workspace_work_item_types(plane, workspace_slug) + if workspace_owns_work_item_types(plane, workspace_slug) + else [] + ) except Exception as exc: - raise RuntimeError( - f"WS3 preclean: failed to delete stale release tag " - f"{EVALUATION_RELEASE_TAG_VERSION!r} id={tag_id}: {exc}" - ) from exc + raise RuntimeError(f"workspace baseline snapshot: list work_item_types failed: {exc}") from exc + baseline["work_item_types"] = { + str(row.id) + for row in rows + if getattr(row, "id", None) is not None + and (is_work_item_type_named(row, BUG_TYPE_NAME) or is_work_item_type_named(row, INCIDENT_TYPE_NAME)) + } + + if "work_item_properties" in wanted: + type_api = getattr(plane, "workspace_work_item_types", None) + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if ( + callable(getattr(type_api, "list", None)) + and callable(getattr(links_api, "list", None)) + and callable(getattr(property_api, "list", None)) + ): + try: + rows = ( + list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME) + if workspace_owns_work_item_types(plane, workspace_slug) + else [] + ) + except Exception as exc: + raise RuntimeError(f"workspace baseline snapshot: list work_item_properties failed: {exc}") from exc + baseline["work_item_properties"] = { + str(row.id) for row in rows if getattr(row, "id", None) is not None and is_severity_property(row) + } + return baseline + +def _raise_fixture_collision(category: str, name: str, object_id: Any) -> None: + raise TaskSkipped( + f"env:fixture-collision:{category}:{name}; pre-existing object id={object_id}; " + "run `python -m evals.cleanup --sentinels --yes`, then retry" + ) + + +def collision_categories(needs: set[str], task_id: str | None) -> set[str]: + categories: set[str] = set() + if "customer" in needs: + categories.update({"customers", "customer_properties"}) + categories.update(_TASK_COLLISION_CATEGORIES.get(task_id or "", set())) + return categories + + +def check_workspace_fixture_collisions( + plane: PlaneClient, + workspace_slug: str, + categories: set[str], +) -> None: + """Reject fixed-name workspace artifacts that would let a no-op agent false-pass. + + Missing API surfaces are silent. List failures raise so an unread workspace is never + mistaken for a clean one. + """ customers = getattr(plane, "customers", None) - properties_api = getattr(customers, "properties", None) if customers is not None else None - if properties_api is not None: + releases = getattr(plane, "releases", None) + specs = ( + ( + "customers", + customers if callable(getattr(customers, "list", None)) else None, + CUSTOMER_NAME, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + ), + ( + "release_tags", + getattr(releases, "tags", None) if releases is not None else None, + EVALUATION_RELEASE_TAG_VERSION, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + ), + ( + "customer_properties", + getattr(customers, "properties", None) if customers is not None else None, + EVALUATION_CUSTOMER_PROPERTY_NAME, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + ), + ) + for category, api, fixture_name, matches in specs: + if category not in categories: + continue + if not callable(getattr(api, "list", None)): + continue try: - page = properties_api.list(workspace_slug=workspace_slug) + rows = list_workspace_rows(api, workspace_slug) except Exception as exc: - raise RuntimeError(f"WS3 preclean: list customer properties failed: {exc}") from exc - rows = page.results if hasattr(page, "results") else page - target = EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() - for customer_property in rows or []: - display = ( - getattr(customer_property, "display_name", None) or getattr(customer_property, "name", None) or "" - ).strip() - if display.casefold() != target: - continue - property_id = getattr(customer_property, "id", None) - if not property_id: - continue + raise RuntimeError(f"workspace fixture collision check: list {category} failed: {exc}") from exc + collision = next((row for row in rows if matches(row)), None) + if collision is not None: + _raise_fixture_collision(category, fixture_name, getattr(collision, "id", "unknown")) + + if "work_item_types" in categories: + api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(api, "list", None)): try: - properties_api.delete(workspace_slug=workspace_slug, property_id=property_id) + if not workspace_owns_work_item_types(plane, workspace_slug): + rows = [] + else: + rows = list_workspace_work_item_types(plane, workspace_slug) + except Exception as exc: + raise RuntimeError(f"workspace fixture collision check: list work_item_types failed: {exc}") from exc + collision = next((row for row in rows if is_work_item_type_named(row, INCIDENT_TYPE_NAME)), None) + if collision is not None: + _raise_fixture_collision( + "work_item_types", + INCIDENT_TYPE_NAME, + getattr(collision, "id", "unknown"), + ) + + if "work_item_properties" in categories: + type_api = getattr(plane, "workspace_work_item_types", None) + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if ( + callable(getattr(type_api, "list", None)) + and callable(getattr(links_api, "list", None)) + and callable(getattr(property_api, "list", None)) + ): + try: + if not workspace_owns_work_item_types(plane, workspace_slug): + rows = [] + else: + rows = list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME) except Exception as exc: raise RuntimeError( - f"WS3 preclean: failed to delete stale customer property " - f"{EVALUATION_CUSTOMER_PROPERTY_NAME!r} id={property_id}: {exc}" + f"workspace fixture collision check: list work_item_properties failed: {exc}" ) from exc + collision = next((row for row in rows if is_severity_property(row)), None) + if collision is not None: + _raise_fixture_collision( + "work_item_properties", + SEVERITY_PROPERTY_NAME, + getattr(collision, "id", "unknown"), + ) -def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) -> dict[str, Any]: +def seed( + plane: PlaneClient, + run_id: str, + needs: set[str], + ctx: dict[str, Any], + *, + task_id: str | None = None, +) -> dict[str, Any]: """Create the eval project and declared fixture groups. Mutates the caller-provided `ctx` in place so project_id is visible to teardown @@ -97,15 +272,13 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) project_name = f"EVAL {run_prefix}" workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] - # Defensive: drop WS3 workspace artifacts that would make a no-op agent pass. - remove_stale_workspace_artifacts(plane, workspace_slug) - # Reset known keys while preserving object identity for the caller. ctx.clear() ctx.update( { "run_id": run_id, "run8": run_prefix, + "task_id": task_id, "workspace_slug": workspace_slug, "project_id": None, "project_name": project_name, @@ -114,6 +287,8 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) "items": {}, "item_identifiers": {}, # title -> PROJ-N for ID-in-hand prompts "item_ids": [], + "fixture_item_ids": {}, # stable fixture title -> API-created id + "fixture_item_titles": {}, # stable fixture title -> per-run display title "state_names": [], # all project state display names (for R1 negative check) "r1_state_name": None, "bug_type": None, @@ -135,16 +310,25 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) "r5_comment_phrases": list(CHECKOUT_COMMENT_PHRASES), "w6_unfinished_titles": list(UNFINISHED_CYCLE_TITLES), "workspace_objects": [], # [{kind, id}, ...] surviving project delete + "randomized_truth": {}, + "evidence_sentinels": {}, + "evidence_targets": {}, + # None means the category was unavailable and name-based teardown must fail closed. + "workspace_baseline": dict.fromkeys(_WORKSPACE_BASELINE_CATEGORIES), } ) - # EV + 4 hex chars; retry with a new suffix on soft-delete identifier collisions. + # Reject only artifacts that could false-pass this task. + task_collision_categories = collision_categories(needs, task_id) + check_workspace_fixture_collisions(plane, workspace_slug, task_collision_categories) + + # EV + 8 hex chars; retry with a new suffix on soft-delete identifier collisions. project = create_project_with_identifier_retry( plane, workspace_slug, name=project_name, identifier_prefix="EV", - initial_suffix=run_prefix[:4].upper(), + initial_suffix=run_prefix.upper(), ) ctx["project_id"] = project.id ctx["project_identifier"] = getattr(project, "identifier", None) @@ -165,11 +349,22 @@ def seed(plane: PlaneClient, run_id: str, needs: set[str], ctx: dict[str, Any]) ctx["ws_feature_exclude"] = sorted(workspace_feature_exclude) # Prior values are captured before the write so teardown restores the workspace # rather than forcing it to whatever this run happened to need. + ownership_categories = set(task_collision_categories) + if "bug_type" in needs: + ownership_categories.update({"work_item_types", "work_item_properties"}) + ctx["workspace_baseline"] = _snapshot_workspace_baseline( + plane, + workspace_slug, + ownership_categories, + ) ctx["workspace_features_prior"] = enable_workspace_features( plane, workspace_slug, exclude=workspace_feature_exclude ) enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) + if task_id == "R7": + seed_r7_state_oracle(plane, workspace_slug, ctx) + # Labels before items so items can attach labels later if needed. if "labels" in needs: seed_labels(plane, workspace_slug, ctx) diff --git a/evals/seed/customers.py b/evals/seed/customers.py index 20678718..4265e9c2 100644 --- a/evals/seed/customers.py +++ b/evals/seed/customers.py @@ -7,11 +7,22 @@ from plane import PlaneClient from plane.models.customers import CreateCustomer, CreateCustomerRequest +from evals.fixtures import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, +) + from .projects import plan_gate_skips -CUSTOMER_NAME = "Acme Corp" -CUSTOMER_REQUEST_NAME = "SSO support" -EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" +__all__ = [ + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "is_evaluation_customer_name", + "seed_customer", +] def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py index 7b4bf4ef..5cb0b004 100644 --- a/evals/seed/cycles.py +++ b/evals/seed/cycles.py @@ -9,10 +9,10 @@ from plane.models.cycles import CreateCycle, UpdateCycle from plane.models.work_items import UpdateWorkItem -from .work_items import PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES +from evals.evidence import set_target_evidence +from evals.fixtures import CYCLE_CURRENT, CYCLE_PAST, PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES -CYCLE_PAST = "Sprint 12" -CYCLE_CURRENT = "Sprint 13" +from .randomize import random_truth_rng, record_randomized_truth def seed_cycles( @@ -29,6 +29,30 @@ def seed_cycles( unachievable and leaves progress_snapshot (a transfer side effect) as the only signal. """ project_id = context["project_id"] + task_id = str(context.get("task_id") or "") + past_name = CYCLE_PAST + current_name = CYCLE_CURRENT + active_fixture_titles = [PAYMENT_WEBHOOK_TITLE, "Session cookie not rotated after login"] + overdue_fixture_title = "Session cookie not rotated after login" + if task_id == "R4": + rng = random_truth_rng(context, "R4:cycles") + current_number = rng.randrange(20, 100) + past_name = f"Sprint {current_number - 1}" + current_name = f"Sprint {current_number}" + active_candidates = [ + title for title in context.get("fixture_item_ids") or {} if title not in set(UNFINISHED_CYCLE_TITLES) + ] + active_fixture_titles = rng.sample(active_candidates, rng.randint(1, min(4, len(active_candidates)))) + overdue_fixture_title = rng.choice(active_fixture_titles) + record_randomized_truth( + context, + "R4.cycle_inventory", + { + "intended_cycle": current_name, + "intended_active_templates": list(active_fixture_titles), + "intended_overdue_template": overdue_fixture_title, + }, + ) me_id = context.get("me_id") or str(plane.users.get_me().id) today = date.today() # Final past window for Sprint 12 after backdate (completedCycles / W6 transfer source). @@ -47,7 +71,7 @@ def seed_cycles( workspace_slug=workspace_slug, project_id=project_id, data=CreateCycle( - name=CYCLE_PAST, + name=past_name, start_date=past_start, end_date=past_end_active, owned_by=me_id, @@ -59,7 +83,7 @@ def seed_cycles( workspace_slug=workspace_slug, project_id=project_id, data=CreateCycle( - name=CYCLE_CURRENT, + name=current_name, start_date=current_start, end_date=current_end, owned_by=me_id, @@ -67,14 +91,17 @@ def seed_cycles( ), ) context["cycles"] = { - CYCLE_PAST: past.id, - CYCLE_CURRENT: current.id, + past_name: past.id, + current_name: current.id, } + context["cycle_past_name"] = past_name + context["cycle_current_name"] = current_name context["cycle_past_id"] = past.id context["cycle_current_id"] = current.id # 2) Add unfinished items to Sprint 12 *before* backdating. - unfinished_ids = [context["items"][title] for title in UNFINISHED_CYCLE_TITLES if title in context["items"]] + fixture_item_ids = context.get("fixture_item_ids") or context.get("items") or {} + unfinished_ids = [fixture_item_ids[title] for title in UNFINISHED_CYCLE_TITLES if title in fixture_item_ids] if unfinished_ids: plane.cycles.add_work_items( workspace_slug=workspace_slug, @@ -84,8 +111,8 @@ def seed_cycles( ) # R4: items on the active cycle (window still open). active_ids: list[str] = [] - for title in (PAYMENT_WEBHOOK_TITLE, "Session cookie not rotated after login"): - item_id = context["items"].get(title) + for title in active_fixture_titles: + item_id = fixture_item_ids.get(title) if item_id: active_ids.append(item_id) if active_ids: @@ -95,7 +122,7 @@ def seed_cycles( cycle_id=current.id, issue_ids=active_ids, ) - overdue_id = context["items"].get("Session cookie not rotated after login") + overdue_id = fixture_item_ids.get(overdue_fixture_title) if overdue_id: plane.work_items.update( workspace_slug=workspace_slug, @@ -103,7 +130,9 @@ def seed_cycles( work_item_id=overdue_id, data=UpdateWorkItem(target_date=(today - timedelta(days=3)).isoformat()), ) - context["r4_overdue_title"] = "Session cookie not rotated after login" + context["r4_overdue_title"] = (context.get("fixture_item_titles") or {}).get( + overdue_fixture_title, overdue_fixture_title + ) context["r4_overdue_id"] = overdue_id context["r4_active_item_ids"] = active_ids @@ -122,3 +151,53 @@ def seed_cycles( context["cycle_past_seed_end_date"] = past_end_active if leave_past_open else past_end_final context["cycle_past_open"] = leave_past_open context["cycle_past_end_date_before_backdate"] = past_end_active + + if task_id == "R4": + confirmed_cycle = plane.cycles.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + ) + confirmed_rows_page = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + ) + confirmed_rows = confirmed_rows_page.results if hasattr(confirmed_rows_page, "results") else confirmed_rows_page + confirmed_active_titles: list[str] = [] + confirmed_overdue_titles: list[str] = [] + for row in confirmed_rows or []: + item_id = getattr(row, "work_item_id", None) or getattr(row, "issue", None) or getattr(row, "id", None) + if hasattr(item_id, "id"): + item_id = item_id.id + if not item_id: + continue + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=str(item_id), + ) + name = str(getattr(detail, "name", None) or "").strip() + if not name: + raise RuntimeError(f"seed R4: active item {item_id} readback has no name") + confirmed_active_titles.append(name) + target_date = str(getattr(detail, "target_date", None) or "")[:10] + if target_date and target_date < today.isoformat(): + confirmed_overdue_titles.append(name) + if not confirmed_active_titles: + raise RuntimeError("seed R4: API readback found no active-cycle items") + context["r4_cycle_name"] = str(getattr(confirmed_cycle, "name", None) or "") + if not context["r4_cycle_name"]: + raise RuntimeError("seed R4: API readback returned an active cycle without a name") + context["r4_active_titles"] = confirmed_active_titles + context["r4_overdue_titles"] = confirmed_overdue_titles + context["randomized_truth"]["R4.cycle_inventory"]["confirmed"] = { + "cycle": context["r4_cycle_name"], + "active_titles": list(confirmed_active_titles), + "overdue_titles": list(confirmed_overdue_titles), + } + set_target_evidence( + context, + [context["r4_cycle_name"], *confirmed_active_titles], + target_ids=[current.id], + ) diff --git a/evals/seed/intake.py b/evals/seed/intake.py index f752f971..78cc5ff5 100644 --- a/evals/seed/intake.py +++ b/evals/seed/intake.py @@ -7,8 +7,7 @@ from plane import PlaneClient from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest -INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" -INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" +from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE def seed_intake(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py index b68ad84b..1e9fc8a3 100644 --- a/evals/seed/item_types.py +++ b/evals/seed/item_types.py @@ -9,6 +9,60 @@ from .projects import is_plan_gate +BUG_TYPE_NAME = "Bug" +INCIDENT_TYPE_NAME = "Incident" +SEVERITY_PROPERTY_NAME = "Severity" + + +def is_work_item_type_named(row: Any, name: str) -> bool: + """Return whether a type row has the exact fixture name, ignoring case/space.""" + return (getattr(row, "name", None) or "").strip().casefold() == name.casefold() + + +def is_severity_property(row: Any) -> bool: + """Return whether a property row is the S1 Severity fixture.""" + display = getattr(row, "display_name", None) or getattr(row, "name", None) or "" + return display.strip().casefold() == SEVERITY_PROPERTY_NAME.casefold() + + +def list_workspace_work_item_types(plane: PlaneClient, workspace_slug: str) -> list[Any]: + """List workspace-owned work-item types using the SDK's non-paginated surface.""" + result = plane.workspace_work_item_types.list(workspace_slug=workspace_slug) + return list((result.results if hasattr(result, "results") else result) or []) + + +def workspace_owns_work_item_types(plane: PlaneClient, workspace_slug: str) -> bool: + """Return the authoritative workspace-vs-project ownership mode for types.""" + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + return bool(dump.get("is_work_item_types_enabled")) + + +def list_workspace_properties_for_type( + plane: PlaneClient, + workspace_slug: str, + type_name: str, +) -> list[Any]: + """Resolve full workspace property rows linked to every type named ``type_name``.""" + item_types = list_workspace_work_item_types(plane, workspace_slug) + target_types = [row for row in item_types if is_work_item_type_named(row, type_name)] + if not target_types: + return [] + + linked_ids: set[str] = set() + for item_type in target_types: + linked = plane.workspace_work_item_types.properties.list( + workspace_slug=workspace_slug, + type_id=item_type.id, + ) + for value in linked or []: + object_id = getattr(value, "id", None) or value + linked_ids.add(str(object_id)) + + properties = plane.workspace_work_item_properties.list(workspace_slug=workspace_slug) + rows = list((properties.results if hasattr(properties, "results") else properties) or []) + return [row for row in rows if getattr(row, "id", None) is not None and str(row.id) in linked_ids] + def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: """Create or resolve a 'Bug' work item type. @@ -17,12 +71,11 @@ def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, A Workspace feature probe uses the real key `is_work_item_types_enabled` (F10). """ project_id = context["project_id"] - target = "Bug" + target = BUG_TYPE_NAME try: - features = plane.workspaces.get_features(workspace_slug=workspace_slug) - dump = features.model_dump() if hasattr(features, "model_dump") else {} - # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional work_item_types key alone. - workspace_owns = bool(dump.get("is_work_item_types_enabled")) + # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional + # work_item_types key alone. + workspace_owns = workspace_owns_work_item_types(plane, workspace_slug) if workspace_owns: existing = next( @@ -74,6 +127,6 @@ def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, A except Exception as exc: if is_plan_gate(exc): context["bug_type"] = None - context["bug_type_skip_reason"] = f"bug_type plan-gated: {exc}" + context["bug_type_skip_reason"] = "env:plan-gated:work-item-types" return raise diff --git a/evals/seed/modules.py b/evals/seed/modules.py index 860de3b1..5c20c425 100644 --- a/evals/seed/modules.py +++ b/evals/seed/modules.py @@ -8,14 +8,9 @@ from plane.models.modules import CreateModule from plane.models.work_items import CreateWorkItem, UpdateWorkItem -from .work_items import find_completed_state, list_states +from evals.fixtures import MODULE_COMPLETED_TITLES, MODULE_NAME -MODULE_NAME = "Checkout revamp" -MODULE_COMPLETED_TITLES = ( - "Module done: cart totals", - "Module done: tax lines", - "Module done: shipping quote", -) +from .work_items import find_completed_state, list_states def seed_module(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: diff --git a/evals/seed/plan.py b/evals/seed/plan.py index 007b0bec..fc23018b 100644 --- a/evals/seed/plan.py +++ b/evals/seed/plan.py @@ -10,7 +10,6 @@ from .releases import RELEASE_NAME from .work_items import ( CHECKOUT_TIMEOUT_TITLE, - DUE_THIS_WEEK_TITLES, PAYMENT_WEBHOOK_TITLE, WORK_ITEM_FIXTURES, ) @@ -22,10 +21,11 @@ def seed_plan(needs: set[str]) -> list[str]: "project: EVAL {run8} (identifier EV{XXXX})", ] if "items" in needs: - lines.append(f"items: {len(WORK_ITEM_FIXTURES)} work items (exactly 4 urgent open)") - lines.append(f" - {PAYMENT_WEBHOOK_TITLE!r} (urgent, non-default started-group state) # R1 target") - lines.append(f" - {len(DUE_THIS_WEEK_TITLES)} assigned-to-me with due this week # R3") - lines.append(f" - comments on {CHECKOUT_TIMEOUT_TITLE!r} # R5 discussion") + lines.append(f"items: {len(WORK_ITEM_FIXTURES)} work items (read truth randomised per row; default 4 urgent)") + lines.append(f" - {PAYMENT_WEBHOOK_TITLE!r} (random started-group state for R1)") + lines.append(" - random assigned-to-me due-this-week selection # R3") + lines.append(f" - random comments on {CHECKOUT_TIMEOUT_TITLE!r} # R5/L2") + lines.append(f" - random attachment count on {PAYMENT_WEBHOOK_TITLE!r} # L5") if "activity_feed" in needs: lines.append( f"activity_feed: gate that activities exist for {CHECKOUT_TIMEOUT_TITLE!r} " @@ -39,7 +39,10 @@ def seed_plan(needs: set[str]) -> list[str]: ) if "cycles" in needs: past_state = "ends tomorrow, still OPEN so it can be closed" if "cycles_open_past" in needs else "past-dated" - lines.append(f"cycles: {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); unfinished on past") + lines.append( + f"cycles: default {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); " + "R4 names/inventory randomised" + ) if "module" in needs: lines.append(f"module: {MODULE_NAME!r} with {len(MODULE_COMPLETED_TITLES)} completed items") if "intake" in needs: @@ -49,7 +52,7 @@ def seed_plan(needs: set[str]) -> list[str]: if "release" in needs: lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") if "second_project" in needs: - lines.append("second_project: EVAL {run8} B with more open Bug-typed items than main (R6)") + lines.append("second_project: EVAL {run8} B with random unequal open Bug counts across both projects (R6)") if "leave_cycles_worklogs_off" in needs: lines.append( "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 74e930bd..b8ce65f4 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -13,8 +13,18 @@ from plane.models.work_items import CreateWorkItem from plane.models.workspaces import WorkspaceFeature -# Soft-deleted projects reserve identifiers; create may 409 — retry with a new suffix. -PROJECT_CREATE_ATTEMPT_LIMIT = 3 +from evals.errors import TaskSkipped +from evals.evidence import set_target_evidence + +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + +# Plane's project identifier field is capped at 12 characters. Keep two characters +# for the eval prefix and use eight hex characters (32 bits), leaving two spare. +PLANE_PROJECT_IDENTIFIER_MAX_LENGTH = 12 +PROJECT_IDENTIFIER_SUFFIX_LENGTH = 8 +# Soft-deleted projects reserve identifiers; a long-lived workspace needs more than +# three chances even with the larger suffix space. +PROJECT_CREATE_ATTEMPT_LIMIT = 8 MAIN_PROJECT_BUG_TITLES = ("Main bug alpha", "Main bug beta") SECOND_PROJECT_BUG_TITLES = ( @@ -55,10 +65,9 @@ def plan_gate_skips(feature: str) -> Iterator[None]: An uncaught seed exception becomes infra_seed and kills the task-rep; a capability the plan excludes is an environment fact, recorded like L2's missing activity worker. - TaskSkipped is imported inside because evals.tasks imports this package at module load. + ``TaskSkipped`` lives in a neutral module, so seed and task packages can import in + either order without a cycle. """ - from evals.tasks.skip import TaskSkipped - try: yield except Exception as exc: @@ -92,16 +101,22 @@ def create_project_with_identifier_retry( """Create a project, regenerating the identifier suffix on soft-delete collisions. Plane soft-deletes reserve identifiers; a 409 (or identifier-in-message error) - triggers a new random 4-char hex suffix. At most three attempts are made, + triggers a new random 8-char hex suffix. At most eight attempts are made, then the last collision error is raised again. """ - suffix = (initial_suffix or "")[:4].upper() - if len(suffix) < 4: - suffix = (suffix + secrets.token_hex(2).upper())[:4] + if len(identifier_prefix) + PROJECT_IDENTIFIER_SUFFIX_LENGTH > PLANE_PROJECT_IDENTIFIER_MAX_LENGTH: + raise ValueError( + f"identifier prefix {identifier_prefix!r} leaves fewer than " + f"{PROJECT_IDENTIFIER_SUFFIX_LENGTH} suffix characters under Plane's " + f"{PLANE_PROJECT_IDENTIFIER_MAX_LENGTH}-character limit" + ) + suffix = (initial_suffix or "")[:PROJECT_IDENTIFIER_SUFFIX_LENGTH].upper() + if len(suffix) < PROJECT_IDENTIFIER_SUFFIX_LENGTH: + suffix = (suffix + secrets.token_hex(4).upper())[:PROJECT_IDENTIFIER_SUFFIX_LENGTH] last_exc: BaseException | None = None for attempt in range(PROJECT_CREATE_ATTEMPT_LIMIT): if attempt > 0: - suffix = secrets.token_hex(2).upper() # 4 hex chars + suffix = secrets.token_hex(4).upper() # 8 hex chars / 32 bits identifier = f"{identifier_prefix}{suffix}" try: return plane.projects.create( @@ -129,8 +144,8 @@ def workspace_feature_state(plane: PlaneClient, workspace_slug: str) -> dict[str """ try: features = plane.workspaces.get_features(workspace_slug=workspace_slug) - except Exception: - return {"customers": None} + except Exception as exc: + raise RuntimeError(f"workspace feature snapshot failed before mutation: {exc}") from exc dump = features.model_dump() if hasattr(features, "model_dump") else {} value = dump.get("customers") if value is None: @@ -204,7 +219,7 @@ def enable_project_features( def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: - """Seed a second project with more open Bug items than the main project.""" + """Seed two API-confirmed Bug counts, randomising R6's winner per row.""" from .item_types import seed_item_type run_prefix = context["run8"] @@ -214,13 +229,11 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s workspace_slug, name=name, identifier_prefix="EB", - initial_suffix=run_prefix[:4].upper(), + initial_suffix=run_prefix.upper(), ) context["second_project_id"] = project.id context["second_project_name"] = name context["second_project_identifier"] = getattr(project, "identifier", None) - # Track for teardown (project delete covers it; still record). - context["second_project_ids"] = [project.id] enable_project_features(plane, workspace_slug, project.id) # Ensure Bug type exists on both projects. @@ -246,9 +259,24 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s raise main_id = context["project_id"] - # Main project: fewer bugs + task_id = str(context.get("task_id") or "") + if task_id == "R6": + rng = random_truth_rng(context, "R6:project-bugs") + hidden_token = random_truth_token(context, "R6:project-bugs") + main_count, second_count = rng.sample(range(1, 6), 2) + main_titles = tuple(f"Main bug case {hidden_token}-{index + 1}" for index in range(main_count)) + second_titles = tuple(f"Second bug case {hidden_token}-{index + 1}" for index in range(second_count)) + record_randomized_truth( + context, + "R6.open_bug_counts", + {"intended_main": main_count, "intended_second": second_count}, + ) + else: + main_titles = MAIN_PROJECT_BUG_TITLES + second_titles = SECOND_PROJECT_BUG_TITLES + main_bug_ids: list[str] = [] - for title in MAIN_PROJECT_BUG_TITLES: + for title in main_titles: item = plane.work_items.create( workspace_slug=workspace_slug, project_id=main_id, @@ -257,15 +285,51 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s main_bug_ids.append(item.id) context["items"][title] = item.id context["item_ids"].append(item.id) - # Second project: more bugs second_bug_ids: list[str] = [] - for title in SECOND_PROJECT_BUG_TITLES: + for title in second_titles: item = plane.work_items.create( workspace_slug=workspace_slug, project_id=project.id, data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] ) second_bug_ids.append(item.id) - context["r6_main_bug_count"] = len(main_bug_ids) - context["r6_second_bug_count"] = len(second_bug_ids) - context["r6_more_bugs_project"] = name # second project has more + if task_id != "R6": + context["r6_main_bug_count"] = len(main_bug_ids) + context["r6_second_bug_count"] = len(second_bug_ids) + context["r6_more_bugs_project"] = name + return + + def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: + count = 0 + for work_item_id in work_item_ids: + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + if str(getattr(detail, "type_id", None) or "") != str(bug_id): + continue + if getattr(detail, "completed_at", None) or getattr(detail, "archived_at", None): + continue + count += 1 + return count + + confirmed_main = confirmed_open_bug_count(main_id, main_bug_ids) + confirmed_second = confirmed_open_bug_count(str(project.id), second_bug_ids) + if confirmed_main == confirmed_second: + raise RuntimeError(f"seed R6: API-confirmed open Bug counts tie ({confirmed_main} each)") + main_project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=main_id) + second_project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project.id) + main_name = str(getattr(main_project, "name", None) or "") + second_name = str(getattr(second_project, "name", None) or "") + if not main_name or not second_name: + raise RuntimeError("seed R6: API project readback returned a project without a name") + context["r6_main_bug_count"] = confirmed_main + context["r6_second_bug_count"] = confirmed_second + context["r6_more_bugs_project"] = main_name if confirmed_main > confirmed_second else second_name + context["randomized_truth"]["R6.open_bug_counts"]["confirmed"] = { + "main": confirmed_main, + "second": confirmed_second, + "winner": context["r6_more_bugs_project"], + } + set_target_evidence(context, [*main_titles, *second_titles], target_ids=[main_id, project.id]) diff --git a/evals/seed/randomize.py b/evals/seed/randomize.py new file mode 100644 index 00000000..855a8462 --- /dev/null +++ b/evals/seed/randomize.py @@ -0,0 +1,44 @@ +"""Per-run hidden-truth randomisation for evaluation fixtures.""" + +from __future__ import annotations + +import hashlib +import random +from typing import Any + + +def random_truth_rng(context: dict[str, Any], namespace: str) -> random.Random: + """Return a reproducible RNG keyed by the full private run id and a namespace. + + Only the run-id prefix appears in the project name shown to the agent. The full id is + persisted on the result row, making a failed fixture reproducible without making its + hidden choices derivable from the prompt. + """ + run_id = str(context.get("run_id") or "") + if not run_id: + raise RuntimeError(f"random truth {namespace}: run_id missing from seed context") + digest = hashlib.sha256(f"{run_id}:{namespace}".encode()).digest() + return random.Random(int.from_bytes(digest, "big")) + + +def random_truth_token(context: dict[str, Any], namespace: str, *, length: int = 10) -> str: + """Return a reproducible hidden token derived from the full private run id. + + Unlike the visible eight-character project prefix, this token depends on the full + run id and a task namespace. It gives response evidence a realistically unique value + without making failed fixture reproduction nondeterministic. + """ + run_id = str(context.get("run_id") or "") + if not run_id: + raise RuntimeError(f"random truth {namespace}: run_id missing from seed context") + if length < 8: + raise ValueError("random truth tokens must contain at least 8 hex characters") + return hashlib.sha256(f"{run_id}:{namespace}:sentinel".encode()).hexdigest()[:length] + + +def record_randomized_truth(context: dict[str, Any], key: str, value: Any) -> None: + """Retain the chosen hidden value in seed context for diagnostics.""" + context.setdefault("randomized_truth", {})[key] = value + + +__all__ = ["random_truth_rng", "random_truth_token", "record_randomized_truth"] diff --git a/evals/seed/releases.py b/evals/seed/releases.py index 342bb5d7..ca2f6cff 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -7,32 +7,78 @@ from plane import PlaneClient from plane.models.releases import CreateRelease, UpdateReleaseChangelog +from evals.changelog import changelog_items, normalize_changelog_text +from evals.evidence import set_target_evidence +from evals.fixtures import ( + EVALUATION_RELEASE_TAG_VERSION, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, +) + from .projects import plan_gate_skips +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth -RELEASE_NAME = "1.2.0" -RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." -EVALUATION_RELEASE_TAG_VERSION = "eval-rc1" +__all__ = [ + "EVALUATION_RELEASE_TAG_VERSION", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "seed_release", +] def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: """Seed the C2 release fixture, skipping the task when the plan excludes releases.""" + task_id = str(context.get("task_id") or "") + release_name = RELEASE_NAME + changelog_text = RELEASE_CHANGELOG_TEXT + if task_id == "C2": + rng = random_truth_rng(context, "C2:release") + hidden_token = random_truth_token(context, "C2:release") + release_name = f"1.{rng.randint(2, 9)}.{rng.randint(0, 20)}-eval.{hidden_token[:8]}" + changelog_text = ( + f"Changelog entry one: OAuth login hardening ticket EVAL-{hidden_token}. " + f"Changelog entry two: webhook retry backoff window {rng.randint(3, 12)}-{hidden_token}." + ) + record_randomized_truth( + context, + "C2.release", + {"intended_name": release_name, "intended_changelog": changelog_text}, + ) with plan_gate_skips("releases"): release = plane.releases.create( workspace_slug=workspace_slug, - data=CreateRelease(name=RELEASE_NAME), + data=CreateRelease(name=release_name), ) - context["release"] = {"id": release.id, "name": RELEASE_NAME} - context["workspace_objects"].append({"kind": "release", "id": release.id}) - # Single changelog body; DESIGN's "2 entries" are encoded as plain-text bullets. - try: + confirmed_release_name = str(getattr(release, "name", None) or "").strip() + if task_id == "C2" and not confirmed_release_name: + raise RuntimeError("release create response did not confirm the randomized release name") + confirmed_release_name = confirmed_release_name or release_name + context["release"] = {"id": release.id, "name": confirmed_release_name} + context["release_name"] = confirmed_release_name + context["workspace_objects"].append({"kind": "release", "id": release.id}) + # Single changelog body; DESIGN's "2 entries" are encoded as plain text. plane.releases.changelog.update( workspace_slug=workspace_slug, release_id=release.id, data=UpdateReleaseChangelog( - description_html=f"

{RELEASE_CHANGELOG_TEXT}

", + description_html=f"

{changelog_text}

", ), ) - except Exception as exc: - # Non-fatal for seed if changelog endpoint is flaky; C2 verifier still checks release name. - print(f"seed warning: release changelog update failed: {exc}") - context["release_changelog_text"] = RELEASE_CHANGELOG_TEXT + confirmed = plane.releases.changelog.retrieve( + workspace_slug=workspace_slug, + release_id=release.id, + ) + confirmed_text = normalize_changelog_text(confirmed) + if not confirmed_text: + raise RuntimeError("release changelog readback was empty after seeding") + context["release_changelog_text"] = confirmed_text + if task_id == "C2": + items = changelog_items(confirmed_text) + if not items: + raise RuntimeError("release changelog readback had no parseable entries after seeding") + context["randomized_truth"]["C2.release"]["confirmed"] = { + "name": confirmed_release_name, + "changelog": confirmed_text, + "items": list(items), + } + set_target_evidence(context, items, target_ids=[release.id]) diff --git a/evals/seed/remove.py b/evals/seed/remove.py index 5fc0232a..00c06bd1 100644 --- a/evals/seed/remove.py +++ b/evals/seed/remove.py @@ -3,18 +3,99 @@ from __future__ import annotations import os +from dataclasses import dataclass from typing import Any from plane import PlaneClient from plane.errors.errors import HttpError from plane.models.workspaces import WorkspaceFeature -from .customers import CUSTOMER_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME +from .customers import CUSTOMER_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME, is_evaluation_customer_name +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, + workspace_owns_work_item_types, +) from .releases import EVALUATION_RELEASE_TAG_VERSION +from .workspace import list_workspace_rows -def _remove_severity_property(plane: PlaneClient, context: dict[str, Any]) -> None: - """Delete Severity properties attached to the seeded Bug type (avoids multi-rep pollution).""" +@dataclass(frozen=True, slots=True) +class CleanupFailure: + """One cleanup operation that failed after teardown attempted it.""" + + operation: str + target: str + error_type: str + message: str + + def __str__(self) -> str: + return f"{self.operation} {self.target}: {self.error_type}: {self.message}" + + +class TeardownError(RuntimeError): + """All cleanup failures from one teardown, raised after every target was attempted.""" + + def __init__(self, failures: list[CleanupFailure]): + self.failures = tuple(failures) + details = "; ".join(str(failure) for failure in self.failures) + super().__init__(f"{len(self.failures)} cleanup operation(s) failed: {details}") + + +def _record_failure( + failures: list[CleanupFailure], + *, + operation: str, + target: Any, + exc: BaseException, +) -> None: + failures.append( + CleanupFailure( + operation=operation, + target=str(target), + error_type=type(exc).__name__, + message=str(exc), + ) + ) + + +def _baseline_ids(ctx: dict[str, Any], category: str) -> set[str] | None: + baseline = ctx.get("workspace_baseline") + if not isinstance(baseline, dict) or baseline.get(category) is None: + return None + return {str(object_id) for object_id in baseline[category]} + + +def _warn_unavailable_baseline( + category: str, + fixture_name: str, + object_ids: list[str], + failures: list[CleanupFailure], +) -> None: + if object_ids: + joined_ids = ", ".join(sorted(object_ids)) + print( + f"teardown warning: {category} baseline unavailable; leaving name-matched {fixture_name!r} ids={joined_ids}" + ) + _record_failure( + failures, + operation="preserve name-matched fixture without baseline", + target=f"{category} {fixture_name!r} ids={joined_ids}", + exc=RuntimeError("workspace baseline unavailable; ownership cannot be determined safely"), + ) + + +def _remove_severity_property( + plane: PlaneClient, + context: dict[str, Any], + failures: list[CleanupFailure], +) -> None: + """Delete only run-owned Severity properties attached to the seeded Bug type.""" bug = context.get("bug_type") if not bug: return @@ -24,10 +105,12 @@ def _remove_severity_property(plane: PlaneClient, context: dict[str, Any]) -> No workspace_slug = context.get("workspace_slug") or "" project_id = context.get("project_id") - properties: list[Any] = [] + # Workspace properties can appear through the project/type endpoint. Resolve both + # scopes and let the workspace scope win for deletion when an ID appears in both. + properties: dict[str, tuple[Any, str]] = {} try: if project_id: - properties = list( + project_properties = list( plane.work_item_properties.list( workspace_slug=workspace_slug, project_id=project_id, @@ -35,58 +118,142 @@ def _remove_severity_property(plane: PlaneClient, context: dict[str, Any]) -> No ) or [] ) + for row in project_properties: + if getattr(row, "id", None) is not None: + properties[str(row.id)] = (row, "project") except HttpError as exc: if exc.status_code not in (404, 405): - print(f"teardown warning: list Severity props failed: {exc}") - return + _record_failure(failures, operation="list", target="Severity properties", exc=exc) except Exception as exc: - print(f"teardown warning: list Severity props failed: {exc}") - return + _record_failure(failures, operation="list", target="Severity properties", exc=exc) - for work_item_property in properties: - display = ( - getattr(work_item_property, "display_name", None) or getattr(work_item_property, "name", None) or "" - ).strip() - if display.lower() != "severity": + if context.get("bug_type_workspace_level"): + try: + for row in list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME): + if getattr(row, "id", None) is not None: + properties[str(row.id)] = (row, "workspace") + except Exception as exc: + _record_failure(failures, operation="list", target="workspace Severity properties", exc=exc) + + baseline = _baseline_ids(context, "work_item_properties") + tracked = { + str(obj.get("id")) + for obj in (context.get("workspace_objects") or []) + if obj.get("kind") == "work_item_property" and obj.get("id") is not None + } + skipped_ids: list[str] = [] + for property_id, (work_item_property, scope) in properties.items(): + if not is_severity_property(work_item_property): continue + if property_id not in tracked: + if baseline is None: + skipped_ids.append(property_id) + continue + if property_id in baseline: + continue try: - if project_id: + if scope == "workspace": + plane.workspace_work_item_properties.delete( + workspace_slug=workspace_slug, + property_id=property_id, + ) + elif project_id: plane.work_item_properties.delete( workspace_slug=workspace_slug, project_id=project_id, type_id=str(bug_type_id), - work_item_property_id=work_item_property.id, + work_item_property_id=property_id, ) - context.setdefault("workspace_objects", []) # no-op anchor except Exception as exc: - print(f"teardown warning: failed to delete Severity property {work_item_property.id}: {exc}") + _record_failure( + failures, + operation="delete Severity property", + target=property_id, + exc=exc, + ) + _warn_unavailable_baseline( + "work item properties", + SEVERITY_PROPERTY_NAME, + skipped_ids, + failures, + ) -def _remove_incident_type(plane: PlaneClient, context: dict[str, Any]) -> None: - """Best-effort cleanup of agent-created Incident type (S3 multi-rep pollution).""" +def _remove_incident_type( + plane: PlaneClient, + context: dict[str, Any], + failures: list[CleanupFailure], +) -> None: + """Clean up S3 Incident types while preserving seed-time workspace ownership.""" + if context.get("task_id") != "S3": + return workspace_slug = context.get("workspace_slug") or "" project_id = context.get("project_id") try: - if context.get("bug_type_workspace_level"): - for item_type in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []: - if (item_type.name or "").strip().casefold() == "incident": - plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=item_type.id) + workspace_owns = workspace_owns_work_item_types(plane, workspace_slug) + except Exception as exc: + _record_failure(failures, operation="detect ownership", target="Incident work item types", exc=exc) + return + + try: + if workspace_owns: + item_types = list_workspace_work_item_types(plane, workspace_slug) + scope = "workspace" elif project_id: - for item_type in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []: - if (item_type.name or "").strip().casefold() == "incident": - plane.work_item_types.delete( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_type_id=item_type.id, - ) + item_types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) + scope = "project" + else: + return except Exception as exc: - print(f"teardown warning: Incident type cleanup failed: {exc}") + _record_failure(failures, operation="list", target="Incident work item types", exc=exc) + return + + for item_type in item_types: + if not is_work_item_type_named(item_type, INCIDENT_TYPE_NAME): + continue + item_type_id = str(item_type.id) + if scope == "workspace": + baseline = _baseline_ids(context, "work_item_types") + tracked = { + str(obj.get("id")) + for obj in (context.get("workspace_objects") or []) + if obj.get("kind") == "work_item_type" and obj.get("id") is not None + } + if item_type_id not in tracked: + if baseline is None: + _warn_unavailable_baseline( + "work item types", + INCIDENT_TYPE_NAME, + [item_type_id], + failures, + ) + continue + if item_type_id in baseline: + continue + try: + if scope == "workspace": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=item_type_id) + else: + assert project_id is not None + plane.work_item_types.delete( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_id=item_type_id, + ) + except Exception as exc: + _record_failure( + failures, + operation="delete Incident work item type", + target=item_type_id, + exc=exc, + ) def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: - """Delete the project and any workspace-scoped objects we created.""" + """Attempt every fixture deletion, then raise all failures as one structured error.""" if not ctx: return + failures: list[CleanupFailure] = [] workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") project_id = ctx.get("project_id") @@ -102,40 +269,56 @@ def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: data=WorkspaceFeature(customers=bool(prior_customers)), ) except Exception as exc: - print(f"teardown warning: restore workspace customers={prior_customers} failed: {exc}") + _record_failure( + failures, + operation="restore workspace customers feature", + target=prior_customers, + exc=exc, + ) # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). try: - _remove_severity_property(plane, ctx) + _remove_severity_property(plane, ctx, failures) except Exception as exc: - print(f"teardown warning: Severity cleanup failed: {exc}") + _record_failure(failures, operation="clean up", target="Severity properties", exc=exc) try: - _remove_incident_type(plane, ctx) + _remove_incident_type(plane, ctx, failures) except Exception as exc: - print(f"teardown warning: Incident cleanup failed: {exc}") + _record_failure(failures, operation="clean up", target="Incident work item types", exc=exc) # Best-effort: agent-created Acme Corp customers (C1) that never hit workspace_objects. try: - page = plane.customers.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page + rows = list_workspace_rows(plane.customers, workspace_slug) + tracked = { + str(obj.get("id")) + for obj in (ctx.get("workspace_objects") or []) + if obj.get("kind") == "customer" and obj.get("id") is not None + } + baseline = _baseline_ids(ctx, "customers") + skipped_ids: list[str] = [] for customer in rows or []: - if (customer.name or "").strip().casefold() in (CUSTOMER_NAME.casefold(), "acme"): - # Only delete if we seeded or created during this run (tracked or name match + run). - tracked = { - obj.get("id") for obj in (ctx.get("workspace_objects") or []) if obj.get("kind") == "customer" - } - if str(customer.id) in tracked or ctx.get("customer") is None: - # Avoid deleting long-lived Acme if we pre-seeded and tracked it — still delete tracked. - if str(customer.id) in tracked or not ctx.get("customer"): - ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": customer.id}) + if not is_evaluation_customer_name(getattr(customer, "name", None)): + continue + customer_id = getattr(customer, "id", None) + if customer_id is None or str(customer_id) in tracked: + continue + if baseline is None: + skipped_ids.append(str(customer_id)) + elif str(customer_id) not in baseline: + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": customer_id}) + _warn_unavailable_baseline("customers", CUSTOMER_NAME, skipped_ids, failures) except Exception as exc: - print(f"teardown warning: customer scan failed: {exc}") + _record_failure(failures, operation="scan", target="evaluation customers", exc=exc) # Workspace-scoped cleanup first (survive project deletion). seen_workspace_objects: set[str] = set() for obj in ctx.get("workspace_objects") or []: - kind = obj.get("kind") - object_id = obj.get("id") + try: + kind = obj.get("kind") + object_id = obj.get("id") + except Exception as exc: + _record_failure(failures, operation="read tracked workspace object", target=repr(obj), exc=exc) + continue if not object_id: continue key = f"{kind}:{object_id}" @@ -155,26 +338,44 @@ def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=object_id) elif kind == "customer_property": plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=object_id) + else: + raise ValueError(f"unsupported workspace object kind {kind!r}") except Exception as exc: - print(f"teardown warning: failed to delete workspace {kind} {object_id}: {exc}") + _record_failure( + failures, + operation=f"delete workspace {kind}", + target=object_id, + exc=exc, + ) # Sweep by well-known WS3 names in case tracking missed an agent-created row. try: - page = plane.releases.tags.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page + rows = list_workspace_rows(plane.releases.tags, workspace_slug) + baseline = _baseline_ids(ctx, "release_tags") + skipped_ids: list[str] = [] for tag in rows or []: if (getattr(tag, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION: tag_id = getattr(tag, "id", None) if tag_id and f"release_tag:{tag_id}" not in seen_workspace_objects: - try: - plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tag_id) - except Exception as exc: - print(f"teardown warning: sweep release tag {tag_id}: {exc}") + if baseline is None: + skipped_ids.append(str(tag_id)) + elif str(tag_id) not in baseline: + try: + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tag_id) + except Exception as exc: + _record_failure( + failures, + operation="sweep release tag", + target=tag_id, + exc=exc, + ) + _warn_unavailable_baseline("release tags", EVALUATION_RELEASE_TAG_VERSION, skipped_ids, failures) except Exception as exc: - print(f"teardown warning: sweep release tags failed: {exc}") + _record_failure(failures, operation="scan", target="evaluation release tags", exc=exc) try: - page = plane.customers.properties.list(workspace_slug=workspace_slug) - rows = page.results if hasattr(page, "results") else page + rows = list_workspace_rows(plane.customers.properties, workspace_slug) + baseline = _baseline_ids(ctx, "customer_properties") + skipped_ids: list[str] = [] target = EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() for customer_property in rows or []: display = ( @@ -183,29 +384,47 @@ def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: if display.casefold() == target: property_id = getattr(customer_property, "id", None) if property_id and f"customer_property:{property_id}" not in seen_workspace_objects: - try: - plane.customers.properties.delete( - workspace_slug=workspace_slug, - property_id=property_id, - ) - except Exception as exc: - print(f"teardown warning: sweep customer property {property_id}: {exc}") + if baseline is None: + skipped_ids.append(str(property_id)) + elif str(property_id) not in baseline: + try: + plane.customers.properties.delete( + workspace_slug=workspace_slug, + property_id=property_id, + ) + except Exception as exc: + _record_failure( + failures, + operation="sweep customer property", + target=property_id, + exc=exc, + ) + _warn_unavailable_baseline( + "customer properties", + EVALUATION_CUSTOMER_PROPERTY_NAME, + skipped_ids, + failures, + ) except Exception as exc: - print(f"teardown warning: sweep customer properties failed: {exc}") + _record_failure(failures, operation="scan", target="evaluation customer properties", exc=exc) # Second project before main (no dependency either way, but be thorough). - for second_project_id in ctx.get("second_project_ids") or []: + second_project_ids = [ctx.get("second_project_id"), *(ctx.get("second_project_ids") or [])] + for second_project_id in dict.fromkeys(second_project_ids): if not second_project_id or second_project_id == project_id: continue try: plane.projects.delete(workspace_slug=workspace_slug, project_id=second_project_id) except Exception as exc: - print(f"teardown warning: failed to delete second project {second_project_id}: {exc}") + _record_failure(failures, operation="delete second project", target=second_project_id, exc=exc) if project_id: try: plane.projects.delete(workspace_slug=workspace_slug, project_id=project_id) except Exception as exc: name = ctx.get("project_name", project_id) - print(f"teardown warning: failed to delete project {name!r}: {exc}") print(f"orphaned project: {name}") + _record_failure(failures, operation="delete project", target=name, exc=exc) + + if failures: + raise TeardownError(failures) diff --git a/evals/seed/states.py b/evals/seed/states.py new file mode 100644 index 00000000..0c03fcce --- /dev/null +++ b/evals/seed/states.py @@ -0,0 +1,57 @@ +"""Project-state fixtures and immutable read oracles.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.states import CreateState + +from evals.evidence import set_target_evidence +from evals.state_oracle import state_name_group_pairs + +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + + +def seed_r7_state_oracle(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Add one hidden state and capture the complete API-confirmed state baseline.""" + project_id = str(context.get("project_id") or "") + if not project_id: + raise RuntimeError("seed R7: project id missing") + rng = random_truth_rng(context, "R7:states") + hidden_token = random_truth_token(context, "R7:states") + state_name = f"Review {hidden_token}" + state_group = rng.choice(("unstarted", "started", "completed")) + created = plane.states.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateState(name=state_name, color="#5E6AD2", group=state_group), + ) + created_id = str(getattr(created, "id", None) or "") + if not created_id: + raise RuntimeError("seed R7: randomized state create returned no id") + + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + rows = list(page.results or []) + if not rows: + raise RuntimeError("seed R7: API readback returned no project states") + pairs = state_name_group_pairs(rows) + expected_pair = f"{state_name} | group: {state_group}" + if expected_pair not in pairs: + raise RuntimeError( + f"seed R7: randomized state missing from API readback; want {expected_pair!r}; have={pairs!r}" + ) + context["r7_state_pairs"] = pairs + context["r7_random_state_id"] = created_id + record_randomized_truth( + context, + "R7.states", + { + "intended": expected_pair, + "confirmed": list(pairs), + }, + ) + set_target_evidence(context, [state_name], target_ids=[project_id]) + + +__all__ = ["seed_r7_state_oracle"] diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index accf0a3a..cda2dc0d 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -2,54 +2,52 @@ from __future__ import annotations +import json from datetime import date, timedelta from typing import Any from plane import PlaneClient +from plane.models.query_params import WorkItemQueryParams +from plane.models.states import CreateState from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem -# Fixed fixture titles for the `items` group. Exactly 4 urgent; the rest medium/high/low. -# "Payment webhook drops retries" is the R1 target (urgent, non-default state). -WORK_ITEM_FIXTURES: list[tuple[str, str]] = [ - ("Payment webhook drops retries", "urgent"), - ("Checkout times out on 3DS challenge", "urgent"), - ("Session cookie not rotated after login", "urgent"), - ("Inventory count goes negative under load", "urgent"), - ("Search results ignore archived projects", "high"), - ("CSV export truncates multi-byte chars", "high"), - ("Webhook secret rotation docs missing", "medium"), - ("Dark mode contrast fails WCAG AA", "medium"), - ("Onboarding email template stale", "medium"), - ("Sidebar collapse flickers on resize", "low"), - ("Tooltip clipped inside modal dialog", "low"), - ("Footer year still says 2024", "none"), -] - -PAYMENT_WEBHOOK_TITLE = WORK_ITEM_FIXTURES[0][0] -# R5 discussion target + distinctive comment phrases (word-boundary matched at verify). -CHECKOUT_TIMEOUT_TITLE = "Checkout times out on 3DS challenge" -CHECKOUT_COMMENT_PHRASES = ( - "stripe callback race", - "retry budget exhausted", -) -# W2 / W3 / W8 targets -SIDEBAR_TITLE = "Sidebar collapse flickers on resize" -DARK_MODE_TITLE = "Dark mode contrast fails WCAG AA" -# W7 relation pair + reference URL -BLOCKING_SOURCE_TITLE = "Search results ignore archived projects" -BLOCKING_TARGET_TITLE = "CSV export truncates multi-byte chars" -BLOCKING_REFERENCE_ADDRESS = "https://example.com/eval/runbook-w7" -# R3: assignees + due this week (seeded count stored in ctx) -DUE_THIS_WEEK_TITLES = ( - "Webhook secret rotation docs missing", - "Onboarding email template stale", -) -# W6 unfinished items in Sprint 12 -UNFINISHED_CYCLE_TITLES = ( - "Inventory count goes negative under load", - "Tooltip clipped inside modal dialog", +from evals.changelog import normalize_changelog_text +from evals.errors import TaskSkipped +from evals.evidence import set_target_evidence +from evals.fixtures import ( + BLOCKING_REFERENCE_ADDRESS, + BLOCKING_SOURCE_TITLE, + BLOCKING_TARGET_TITLE, + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DARK_MODE_TITLE, + DUE_THIS_WEEK_TITLES, + PAYMENT_WEBHOOK_TITLE, + SIDEBAR_TITLE, + UNFINISHED_CYCLE_TITLES, + WORK_ITEM_FIXTURES, ) +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + +__all__ = [ + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "DARK_MODE_TITLE", + "DUE_THIS_WEEK_TITLES", + "PAYMENT_WEBHOOK_TITLE", + "SIDEBAR_TITLE", + "UNFINISHED_CYCLE_TITLES", + "WORK_ITEM_FIXTURES", + "find_completed_state", + "list_states", + "require_activities", + "seed_work_items", +] + def list_states(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) @@ -67,8 +65,83 @@ def find_completed_state(states: list[Any]) -> Any | None: return completed[0] +def _enum_value(value: Any) -> str: + return str(getattr(value, "value", value) or "") + + +def _as_id(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, dict): + return str(value.get("id") or "") + return str(getattr(value, "id", None) or "") + + +def _list_all_work_items(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: + rows: list[Any] = [] + cursor: str | None = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + rows.extend(page.results or []) + if not page.next_page_results: + return rows + cursor = page.next_cursor + + +def _resolve_state_name( + plane: PlaneClient, + workspace_slug: str, + project_id: str, + state_ref: Any, +) -> str: + direct = getattr(state_ref, "name", None) or (state_ref.get("name") if isinstance(state_ref, dict) else None) + if direct: + return str(direct) + state_id = _as_id(state_ref) + if not state_id: + return "" + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return str(state.name or "") + + +def _confirm_open_urgent_items(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[str]: + states = list_states(plane, workspace_slug, project_id) + closed = { + str(state.id) + for state in states + if _enum_value(getattr(state, "group", None)).casefold() in {"completed", "cancelled"} + } + titles: list[str] = [] + for item in _list_all_work_items(plane, workspace_slug, project_id): + if _enum_value(getattr(item, "priority", None)).casefold() != "urgent": + continue + if _as_id(getattr(item, "state", None)) in closed: + continue + title = str(getattr(item, "name", None) or "").strip() + if not title: + raise RuntimeError(f"seed R2: API readback returned urgent item without a name: {item!r}") + titles.append(title) + return titles + + +def _serialized_rows(rows: list[Any]) -> str: + payload = [row.model_dump(mode="json") if hasattr(row, "model_dump") else row for row in rows] + return json.dumps(payload, default=str, ensure_ascii=False) + + +def _comment_text(comment: Any) -> str: + stripped = str(getattr(comment, "comment_stripped", None) or "").strip() + if stripped: + return " ".join(stripped.split()) + return normalize_changelog_text(str(getattr(comment, "comment_html", None) or "")) + + def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: project_id = context["project_id"] + task_id = str(context.get("task_id") or "") + rng = random_truth_rng(context, f"{task_id or 'shared'}:work-items") + hidden_token = random_truth_token(context, f"{task_id or 'shared'}:work-items") states = list_states(plane, workspace_slug, project_id) context["state_names"] = sorted({(state.name or "").strip() for state in states if (state.name or "").strip()}) @@ -83,7 +156,29 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, "seed items: no started-group state available to place the R1 target; " f"states={[(state.name, state.group, state.default) for state in states]}" ) - r1_state = started[0] + base_started_state = started[0] + state_targets: dict[str, Any] = {PAYMENT_WEBHOOK_TITLE: base_started_state} + if task_id in {"R1", "I2"}: + random_state_name = f"Investigating {hidden_token}" + random_state = plane.states.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateState( + name=random_state_name, + color="#5E6AD2", + group="started", + ), + ) + if not getattr(random_state, "id", None): + raise RuntimeError(f"seed {task_id}: random state create returned no id") + state_targets[PAYMENT_WEBHOOK_TITLE if task_id == "R1" else SIDEBAR_TITLE] = random_state + record_randomized_truth( + context, + f"{task_id}.state", + {"intended": random_state_name, "created_id": str(random_state.id)}, + ) + + r1_state = state_targets[PAYMENT_WEBHOOK_TITLE] context["r1_state_name"] = r1_state.name context["r1_state_id"] = r1_state.id @@ -97,12 +192,31 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, due_this_week = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)).isoformat() context["r3_due_date"] = due_this_week + urgent_target = rng.randint(2, 7) if task_id == "R2" else 4 + if task_id == "R2": + record_randomized_truth(context, "R2.urgent_open_count", {"intended": urgent_target}) + + r3_templates: set[str] = set(DUE_THIS_WEEK_TITLES) + if task_id == "R3": + r3_count = rng.randint(1, 4) + candidates = [title for title, _priority in WORK_ITEM_FIXTURES] + r3_templates = set(rng.sample(candidates, r3_count)) + record_randomized_truth(context, "R3.due_templates", sorted(r3_templates)) + + randomize_titles = task_id in {"R2", "R3", "R4"} urgent_count = 0 - for title, priority in WORK_ITEM_FIXTURES: + for index, (fixture_title, fixture_priority) in enumerate(WORK_ITEM_FIXTURES): + title = ( + f"{fixture_title} · case {hidden_token}-{index + 1}" + if randomize_titles and (task_id in {"R2", "R4"} or fixture_title in r3_templates) + else fixture_title + ) + priority = "urgent" if index < urgent_target else ("high" if fixture_priority == "urgent" else fixture_priority) data_kwargs: dict[str, Any] = {"name": title, "priority": priority} - if title == PAYMENT_WEBHOOK_TITLE: - data_kwargs["state"] = str(r1_state.id) - if title in DUE_THIS_WEEK_TITLES: + target_state = state_targets.get(fixture_title) + if target_state is not None: + data_kwargs["state"] = str(target_state.id) + if fixture_title in r3_templates: data_kwargs["assignees"] = [me_id] data_kwargs["target_date"] = due_this_week item = plane.work_items.create( @@ -111,57 +225,207 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, data=CreateWorkItem(**data_kwargs), # type: ignore[arg-type] ) # Some APIs ignore state on create; force via update if needed. - if title == PAYMENT_WEBHOOK_TITLE: + if target_state is not None: current = getattr(item, "state", None) current_id = current if isinstance(current, str) else getattr(current, "id", None) - if str(current_id) != str(r1_state.id): + if str(current_id) != str(target_state.id): item = plane.work_items.update( workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id, - data=UpdateWorkItem(state=str(r1_state.id)), + data=UpdateWorkItem(state=str(target_state.id)), ) context["items"][title] = item.id + context["fixture_item_ids"][fixture_title] = item.id + context["fixture_item_titles"][fixture_title] = title context["item_ids"].append(item.id) sequence = getattr(item, "sequence_id", None) if sequence is not None and context.get("project_identifier"): - context["item_identifiers"][title] = f"{context['project_identifier']}-{sequence}" + context["item_identifiers"][fixture_title] = f"{context['project_identifier']}-{sequence}" if priority == "urgent": urgent_count += 1 - assert urgent_count == 4, f"fixture invariant: expected 4 urgent items, got {urgent_count}" + assert urgent_count == urgent_target, ( + f"fixture invariant: expected {urgent_target} urgent items, got {urgent_count}" + ) # R5: seed discussion comments on the known item. - target_id = context["items"].get(CHECKOUT_TIMEOUT_TITLE) + target_id = context["fixture_item_ids"].get(CHECKOUT_TIMEOUT_TITLE) + comment_phrases = list(CHECKOUT_COMMENT_PHRASES) + if task_id in {"R5", "L2"}: + comment_count = rng.randint(1, 4) + comment_phrases = [ + f"{CHECKOUT_COMMENT_PHRASES[index % len(CHECKOUT_COMMENT_PHRASES)]} ref-{hidden_token}-{index + 1}" + for index in range(comment_count) + ] + truth_key = "R5.comments" if task_id == "R5" else "L2.activity_count" + truth_value = ( + {"intended": list(comment_phrases)} if task_id == "R5" else {"intended_comment_count": comment_count} + ) + record_randomized_truth(context, truth_key, truth_value) + if task_id == "L2": + context["l2_comment_phrases"] = list(comment_phrases) + comment_ids: set[str] = set() if target_id: - for phrase in CHECKOUT_COMMENT_PHRASES: - plane.work_items.comments.create( + for phrase in comment_phrases: + created_comment = plane.work_items.comments.create( workspace_slug=workspace_slug, project_id=project_id, work_item_id=target_id, data=CreateWorkItemComment(comment_html=f"

{phrase}

"), ) + if getattr(created_comment, "id", None) is not None: + comment_ids.add(str(created_comment.id)) + + # Capture every affected read oracle from API-confirmed state, never from the random choice. + if task_id in {"R1", "I2"}: + target_fixture = PAYMENT_WEBHOOK_TITLE if task_id == "R1" else SIDEBAR_TITLE + target_id = context["fixture_item_ids"][target_fixture] + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + ) + confirmed_name = _resolve_state_name(plane, workspace_slug, project_id, detail.state) + if not confirmed_name: + raise RuntimeError(f"seed {task_id}: API readback could not resolve target state") + oracle_key = "r1_state_name" if task_id == "R1" else "i2_state_name" + context[oracle_key] = confirmed_name + context["randomized_truth"][f"{task_id}.state"]["confirmed"] = confirmed_name + set_target_evidence(context, [confirmed_name], target_ids=[target_id]) + + if task_id == "R2": + confirmed_titles = _confirm_open_urgent_items(plane, workspace_slug, project_id) + if not confirmed_titles: + raise RuntimeError("seed R2: API readback found no urgent open work items") + context["r2_urgent_open_count"] = len(confirmed_titles) + context["randomized_truth"]["R2.urgent_open_count"]["confirmed"] = len(confirmed_titles) + set_target_evidence(context, confirmed_titles, target_ids=[project_id]) + + if task_id == "R3": + confirmed_due_titles: list[str] = [] + week_start = today - timedelta(days=today.weekday()) + week_end = week_start + timedelta(days=6) + for item_id in context["item_ids"]: + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item_id, + ) + target_date = str(getattr(detail, "target_date", None) or "")[:10] + assignee_ids = {_as_id(value) for value in (getattr(detail, "assignees", None) or [])} + if target_date and week_start.isoformat() <= target_date <= week_end.isoformat() and me_id in assignee_ids: + confirmed_due_titles.append(str(detail.name)) + if not confirmed_due_titles: + raise RuntimeError("seed R3: API readback found no assigned due-this-week items") + context["r3_due_titles"] = confirmed_due_titles + context["r3_due_count"] = len(confirmed_due_titles) + context["randomized_truth"]["R3.due_templates"] = { + "intended": sorted(r3_templates), + "confirmed": { + "titles": list(confirmed_due_titles), + "count": len(confirmed_due_titles), + }, + } + set_target_evidence(context, confirmed_due_titles, target_ids=[project_id]) + + if task_id == "R5": + page = plane.work_items.comments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + ) + confirmed_comments = [ + _comment_text(comment) + for comment in (page.results or []) + if not comment_ids or str(getattr(comment, "id", "")) in comment_ids + ] + confirmed_comments = [text for text in confirmed_comments if text] + if len(confirmed_comments) != len(comment_phrases): + raise RuntimeError( + f"seed R5: API readback returned {len(confirmed_comments)} eval comments; want {len(comment_phrases)}" + ) + context["r5_comment_phrases"] = confirmed_comments + context["randomized_truth"]["R5.comments"]["confirmed"] = list(confirmed_comments) + set_target_evidence(context, confirmed_comments, target_ids=[target_id]) + + if task_id == "L1": + work_item_id = str(context["fixture_item_ids"].get(PAYMENT_WEBHOOK_TITLE) or "") + if not work_item_id: + raise RuntimeError("seed L1: target work item id missing") + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + confirmed_id = str(getattr(detail, "id", None) or "") + if confirmed_id != work_item_id: + raise RuntimeError(f"seed L1: target work item readback id={confirmed_id!r}; want {work_item_id!r}") + context["l1_expected_summary_ids"] = [confirmed_id] + set_target_evidence(context, [confirmed_id], target_ids=[project_id]) + + if task_id == "L5": + attachment_target_id = context["fixture_item_ids"][PAYMENT_WEBHOOK_TITLE] + attachment_count = rng.randint(1, 3) + record_randomized_truth(context, "L5.attachment_count", {"intended": attachment_count}) + intended_names: list[str] = [] + for index in range(attachment_count): + name = f"diagnostic-{hidden_token}-{index + 1}.txt" + intended_names.append(name) + plane.work_items.attachments.upload_from_bytes( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=attachment_target_id, + file_bytes=f"eval attachment {hidden_token}-{index + 1}\n".encode(), + name=name, + content_type="text/plain", + ) + attachments = plane.work_items.attachments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=attachment_target_id, + ) + confirmed_rows = list(attachments.results or []) + confirmed_attachment_count = len(confirmed_rows) + context["l5_attachment_count"] = confirmed_attachment_count + context["randomized_truth"]["L5.attachment_count"]["confirmed"] = confirmed_attachment_count + response_blob = _serialized_rows(confirmed_rows) + confirmed_names = [name for name in intended_names if name in response_blob] + if len(confirmed_names) != attachment_count: + raise RuntimeError( + f"seed L5: attachment readback exposed {len(confirmed_names)} randomized names; want {attachment_count}" + ) + set_target_evidence(context, confirmed_names, target_ids=[attachment_target_id]) def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: - """Skip L2 when comments never materialize as activities (no activity worker). + """Require L2's seeded comments to materialize as activities. - Raises :class:`evals.tasks.TaskSkipped` with reason ``env:no-activity-worker`` - so the harness records a skip, not a task failure. + Only a successful, empty read evidences a missing activity worker and becomes the + expected ``env:no-activity-worker`` capability skip. Missing fixture identifiers and + inconsistent readback are fixture errors; API read failures propagate as infrastructure. """ - from evals.tasks.skip import TaskSkipped - project_id = context.get("project_id") work_item_id = (context.get("items") or {}).get(CHECKOUT_TIMEOUT_TITLE) if not project_id or not work_item_id: - raise TaskSkipped("env:no-activity-worker") - try: - page = plane.work_items.activities.list( - workspace_slug=workspace_slug, - project_id=project_id, - work_item_id=work_item_id, - ) - except Exception as exc: - raise TaskSkipped(f"env:no-activity-worker ({type(exc).__name__}: {exc})") from exc + missing = [name for name, value in (("project_id", project_id), ("work_item_id", work_item_id)) if not value] + raise RuntimeError(f"seed L2 fixture error: missing {', '.join(missing)}") + page = plane.work_items.activities.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) rows = page.results if hasattr(page, "results") else page - if len(list(rows or [])) < 1: + activity_rows = list(rows or []) + activity_count = len(activity_rows) + if activity_count < 1: raise TaskSkipped("env:no-activity-worker") + context["l2_activity_count"] = activity_count + if str(context.get("task_id") or "") == "L2": + randomised = context.setdefault("randomized_truth", {}).setdefault("L2.activity_count", {}) + randomised["confirmed"] = activity_count + candidates = [str(value) for value in context.get("l2_comment_phrases") or []] + response_blob = _serialized_rows(activity_rows) + visible = [value for value in candidates if value in response_blob] + if not visible: + raise RuntimeError("seed L2 fixture error: activity readback omitted randomized comment evidence") + set_target_evidence(context, visible, target_ids=[work_item_id]) diff --git a/evals/seed/workspace.py b/evals/seed/workspace.py new file mode 100644 index 00000000..4e59e455 --- /dev/null +++ b/evals/seed/workspace.py @@ -0,0 +1,22 @@ +"""Shared helpers for workspace-scoped evaluation fixtures.""" + +from __future__ import annotations + +from typing import Any + +from plane.models.query_params import PaginatedQueryParams + + +def list_workspace_rows(api: Any, workspace_slug: str) -> list[Any]: + """List every row from a paginated workspace-scoped API.""" + rows: list[Any] = [] + cursor = None + while True: + page = api.list( + workspace_slug=workspace_slug, + params=PaginatedQueryParams(per_page=100, cursor=cursor), + ) + rows.extend((page.results if hasattr(page, "results") else page) or []) + if not getattr(page, "next_page_results", False): + return rows + cursor = page.next_cursor diff --git a/tests/evals/seed/test_gate_tolerance.py b/tests/evals/seed/test_gate_tolerance.py index e2fe1250..b09a1f1a 100644 --- a/tests/evals/seed/test_gate_tolerance.py +++ b/tests/evals/seed/test_gate_tolerance.py @@ -14,6 +14,7 @@ import pytest from plane.errors.errors import HttpError +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import seed_customer, seed_release from evals.tasks.skip import TaskSkipped @@ -88,3 +89,131 @@ def _refuse(**_kwargs: Any): assert created == ["customer"] assert "customer_request" not in context + + +def test_release_changelog_write_behaviours(): + cases = ( + ("write failure is fatal", RuntimeError("connection reset"), RuntimeError, None), + ("plan gate remains a skip", PLAN_REFUSAL, TaskSkipped, "env:plan-gated:releases"), + ) + for label, error, expected_error, expected_reason in cases: + with pytest.MonkeyPatch.context(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda error=error, **kw: (_ for _ in ()).throw(error), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(expected_error) as caught: + seed_release(plane, "ws", context) + + if expected_reason is not None: + assert caught.value.reason == expected_reason, label + assert context["release"]["id"] == "release-1", label + assert context["workspace_objects"] == [{"kind": "release", "id": "release-1"}], label + assert "release_changelog_text" not in context, label + + +def test_release_changelog_readback_sets_the_api_confirmed_baseline(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: SimpleNamespace( + description_html="

Changelog entry one: API-confirmed fact.

", + ), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + + seed_release(plane, "ws", context) + + assert context["release_changelog_text"] == "Changelog entry one: API-confirmed fact." + + +def test_c2_release_truth_is_randomized_and_api_confirmed(): + contexts: list[dict[str, Any]] = [] + for run_id in ("c2000000aaaaaaaa", "c2000000bbbbbbbb"): + stored: dict[str, str] = {} + + def create(*, data, _stored=stored, _run_id=run_id, **kwargs): + _stored["name"] = data.name + return SimpleNamespace(id=f"release-{_run_id[-4:]}", name=data.name) + + def update(*, data, _stored=stored, **kwargs): + _stored["html"] = data.description_html + + def retrieve(*, _stored=stored, **kwargs): + return SimpleNamespace(description_html=_stored["html"]) + + plane = SimpleNamespace( + releases=SimpleNamespace( + create=create, + changelog=SimpleNamespace( + update=update, + retrieve=retrieve, + ), + ) + ) + context: dict[str, Any] = { + "run_id": run_id, + "task_id": "C2", + "workspace_objects": [], + "randomized_truth": {}, + } + seed_release(plane, "ws", context) + contexts.append(context) + + assert context["release"]["name"] == stored["name"] + assert context["randomized_truth"]["C2.release"]["confirmed"]["changelog"] == context["release_changelog_text"] + assert context["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + assert contexts[0]["release_name"] != contexts[1]["release_name"] + assert contexts[0]["release_changelog_text"] != contexts[1]["release_changelog_text"] + + +@pytest.mark.parametrize( + ("error", "expected_error", "expected_reason"), + [ + (RuntimeError("readback failed"), RuntimeError, None), + (PLAN_REFUSAL, TaskSkipped, "env:plan-gated:releases"), + ], +) +def test_release_changelog_readback_failures(error, expected_error, expected_reason): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: (_ for _ in ()).throw(error), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + + with pytest.raises(expected_error) as caught: + seed_release(plane, "ws", context) + + if expected_reason is not None: + assert caught.value.reason == expected_reason + assert "release_changelog_text" not in context + + +def test_empty_release_changelog_readback_is_a_seed_failure(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: SimpleNamespace(description_html="

"), + ), + ) + ) + + with pytest.raises(RuntimeError, match="readback was empty after seeding"): + seed_release(plane, "ws", {"workspace_objects": []}) diff --git a/tests/evals/seed/test_plan_gate.py b/tests/evals/seed/test_plan_gate.py index 63381338..886a6ba6 100644 --- a/tests/evals/seed/test_plan_gate.py +++ b/tests/evals/seed/test_plan_gate.py @@ -1,7 +1,7 @@ """Characterization of `is_plan_gate` against the payloads `api/` actually returns. A gate becomes an environment skip and anything else stays a real error, so a -misclassification here either hides a defect or invents one. +a wrong category here either hides a defect or invents one. """ from __future__ import annotations diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py new file mode 100644 index 00000000..331f67ce --- /dev/null +++ b/tests/evals/seed/test_read_randomization.py @@ -0,0 +1,240 @@ +"""Focused seed/readback tests for per-row read-task truth.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.seed.cycles import seed_cycles +from evals.seed.states import seed_r7_state_oracle +from evals.seed.work_items import require_activities, seed_work_items + + +class _Page: + def __init__(self, results: list[Any]): + self.results = results + self.next_page_results = False + self.next_cursor = None + + +class _ReadSeedPlane: + def __init__(self): + self._states = [ + SimpleNamespace(id="state-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="state-done", name="Done", group="completed", default=False), + ] + self._items: dict[str, SimpleNamespace] = {} + self._comments: dict[str, list[SimpleNamespace]] = {} + self._attachments: dict[str, list[SimpleNamespace]] = {} + self._cycles: dict[str, SimpleNamespace] = {} + self._cycle_items: dict[str, list[str]] = {} + self.states = SimpleNamespace( + list=lambda **kwargs: _Page(list(self._states)), + create=self._create_state, + retrieve=self._retrieve_state, + ) + self.users = SimpleNamespace(get_me=lambda: SimpleNamespace(id="user-me")) + self.work_items = SimpleNamespace( + create=self._create_item, + update=self._update_item, + retrieve=self._retrieve_item, + list=lambda **kwargs: _Page(list(self._items.values())), + comments=SimpleNamespace(create=self._create_comment, list=self._list_comments), + activities=SimpleNamespace(list=self._list_activities), + attachments=SimpleNamespace(upload_from_bytes=self._upload_attachment, list=self._list_attachments), + ) + self.cycles = SimpleNamespace( + create=self._create_cycle, + update=self._update_cycle, + retrieve=self._retrieve_cycle, + add_work_items=self._add_cycle_items, + list_work_items=self._list_cycle_items, + ) + + def _create_state(self, *, data, **kwargs): + state = SimpleNamespace( + id=f"state-{len(self._states) + 1}", + name=data.name, + group=data.group, + default=False, + ) + self._states.append(state) + return state + + def _retrieve_state(self, *, state_id, **kwargs): + return next(state for state in self._states if state.id == state_id) + + def _create_item(self, *, data, **kwargs): + work_item_id = f"item-{len(self._items) + 1}" + item = SimpleNamespace( + id=work_item_id, + sequence_id=len(self._items) + 1, + name=data.name, + priority=data.priority, + state=data.state or "state-started", + target_date=data.target_date, + assignees=list(data.assignees or []), + ) + self._items[work_item_id] = item + return item + + def _update_item(self, *, work_item_id, data, **kwargs): + item = self._items[work_item_id] + for key, value in data.model_dump(exclude_none=True).items(): + setattr(item, key, value) + return item + + def _retrieve_item(self, *, work_item_id, **kwargs): + return self._items[work_item_id] + + def _create_comment(self, *, work_item_id, data, **kwargs): + rows = self._comments.setdefault(work_item_id, []) + comment = SimpleNamespace( + id=f"comment-{len(rows) + 1}", + comment_html=data.comment_html, + comment_stripped=None, + ) + rows.append(comment) + return comment + + def _list_comments(self, *, work_item_id, **kwargs): + return _Page(list(self._comments.get(work_item_id, []))) + + def _list_activities(self, *, work_item_id, **kwargs): + rows = [ + SimpleNamespace(id=f"activity-{row.id}", comment=row.comment_html) + for row in self._comments.get(work_item_id, []) + ] + return _Page(rows) + + def _upload_attachment(self, *, work_item_id, name, **kwargs): + rows = self._attachments.setdefault(work_item_id, []) + attachment = SimpleNamespace(id=f"attachment-{len(rows) + 1}", name=name) + rows.append(attachment) + return attachment + + def _list_attachments(self, *, work_item_id, **kwargs): + return _Page(list(self._attachments.get(work_item_id, []))) + + def _create_cycle(self, *, data, **kwargs): + cycle_id = f"cycle-{len(self._cycles) + 1}" + cycle = SimpleNamespace(id=cycle_id, name=data.name, end_date=data.end_date) + self._cycles[cycle_id] = cycle + self._cycle_items[cycle_id] = [] + return cycle + + def _update_cycle(self, *, cycle_id, data, **kwargs): + cycle = self._cycles[cycle_id] + cycle.end_date = data.end_date + return cycle + + def _retrieve_cycle(self, *, cycle_id, **kwargs): + return self._cycles[cycle_id] + + def _add_cycle_items(self, *, cycle_id, issue_ids, **kwargs): + self._cycle_items[cycle_id].extend(str(value) for value in issue_ids) + + def _list_cycle_items(self, *, cycle_id, **kwargs): + return _Page([SimpleNamespace(work_item_id=value) for value in self._cycle_items[cycle_id]]) + + +def _context(task_id: str, run_id: str) -> dict[str, Any]: + return { + "run_id": run_id, + "run8": run_id[:8], + "task_id": task_id, + "project_id": "project-1", + "project_identifier": "EVTEST", + "items": {}, + "item_ids": [], + "item_identifiers": {}, + "fixture_item_ids": {}, + "fixture_item_titles": {}, + "randomized_truth": {}, + } + + +@pytest.mark.parametrize( + ("task_id", "oracle_key", "truth_key"), + [ + ("R1", "r1_state_name", "R1.state"), + ("R2", "r2_urgent_open_count", "R2.urgent_open_count"), + ("R3", "r3_due_titles", "R3.due_templates"), + ("R5", "r5_comment_phrases", "R5.comments"), + ("I2", "i2_state_name", "I2.state"), + ("L2", "l2_activity_count", "L2.activity_count"), + ("L5", "l5_attachment_count", "L5.attachment_count"), + ], +) +def test_work_item_read_truth_is_randomized_and_api_confirmed(task_id, oracle_key, truth_key): + plane = _ReadSeedPlane() + ctx = _context(task_id, f"{task_id.lower():0<8}0123456789abcdef") + seed_work_items(plane, "ws", ctx) + if task_id == "L2": + require_activities(plane, "ws", ctx) + + assert ctx[oracle_key] not in (None, "", []) + assert "confirmed" in ctx["randomized_truth"][truth_key] + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + +def test_r2_randomized_counts_differ_between_rows_after_api_readback(): + contexts = [] + for run_id in ("00000000aaaaaaaa", "11111111bbbbbbbb"): + plane = _ReadSeedPlane() + ctx = _context("R2", run_id) + seed_work_items(plane, "ws", ctx) + contexts.append(ctx) + + counts = [ctx["r2_urgent_open_count"] for ctx in contexts] + assert counts == [6, 3] + for ctx in contexts: + truth = ctx["randomized_truth"]["R2.urgent_open_count"] + assert truth["confirmed"] == ctx["r2_urgent_open_count"] + + +def test_r4_cycle_inventory_is_randomized_and_api_confirmed(): + plane = _ReadSeedPlane() + ctx = _context("R4", "44444444aaaaaaaa") + seed_work_items(plane, "ws", ctx) + seed_cycles(plane, "ws", ctx) + + truth = ctx["randomized_truth"]["R4.cycle_inventory"] + assert ctx["r4_cycle_name"].startswith("Sprint ") + assert ctx["r4_cycle_name"] != "Sprint 13" + assert ctx["r4_active_titles"] + assert ctx["r4_overdue_titles"] + assert truth["confirmed"] == { + "cycle": ctx["r4_cycle_name"], + "active_titles": ctx["r4_active_titles"], + "overdue_titles": ctx["r4_overdue_titles"], + } + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + +def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): + contexts: list[dict[str, Any]] = [] + for run_id in ("77777777aaaaaaaa", "88888888bbbbbbbb"): + plane = _ReadSeedPlane() + ctx = _context("R7", run_id) + seed_r7_state_oracle(plane, "ws", ctx) + contexts.append(ctx) + + assert contexts[0]["r7_state_pairs"] != contexts[1]["r7_state_pairs"] + for ctx in contexts: + truth = ctx["randomized_truth"]["R7.states"] + assert truth["confirmed"] == ctx["r7_state_pairs"] + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + +def test_l1_seed_oracle_is_the_api_confirmed_target_id(): + plane = _ReadSeedPlane() + ctx = _context("L1", "11111111cccccccc") + + seed_work_items(plane, "ws", ctx) + + assert ctx["l1_expected_summary_ids"] == [ctx["fixture_item_ids"]["Payment webhook drops retries"]] + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == tuple(ctx["l1_expected_summary_ids"]) diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index 1aa60315..9cbc26ae 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import inspect from types import SimpleNamespace from typing import Any @@ -12,15 +13,21 @@ from evals import cleanup as cleanup_mod from evals import seed as seed_mod +from evals.errors import TaskSkipped +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import ( + R5_TITLE, create_project_with_identifier_retry, is_identifier_collision, seed_plan, + seed_second_project, ) from evals.tasks.debias import ( L3_TAG_VERSION, L4_PROP_DISPLAY, ) +from evals.tasks.read import verify_r6 +from tests.evals.conftest import case_params class _Page: @@ -57,362 +64,599 @@ def __init__(self): delete=lambda **kw: None, ) self.projects = SimpleNamespace(delete=lambda **kw: None) + self.work_item_types = SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None) self.workspace_work_item_types = SimpleNamespace(delete=lambda **kw: None) self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) -def test_seed_behaviours(monkeypatch): - def test_seed_plan_behaviours(): - def test_seed_plan_covers_all_groups(): - groups = { - "items", - "labels", - "bug_type", - "cycles", - "module", - "intake", - "customer", - "release", - "second_project", +def test_r6_random_truth_is_per_seed_and_oracle_is_api_confirmed(): + class Projects: + def __init__(self, main_id: str, main_name: str): + self.names = {main_id: main_name} + + def create(self, workspace_slug, data): + project_id = f"second-{len(self.names)}" + self.names[project_id] = data.name + return SimpleNamespace(id=project_id, name=data.name, identifier=data.identifier) + + def update(self, **kwargs): + return None + + def update_features(self, **kwargs): + return None + + def retrieve(self, *, project_id, **kwargs): + return SimpleNamespace(id=project_id, name=self.names[project_id]) + + class WorkItems: + def __init__(self, *, mark_second_non_bug: int): + self.rows: dict[str, SimpleNamespace] = {} + self.project_ids: dict[str, list[str]] = {} + self.mark_second_non_bug = mark_second_non_bug + + def create(self, *, project_id, data, **kwargs): + project_rows = self.project_ids.setdefault(str(project_id), []) + work_item_id = f"{project_id}-wi-{len(project_rows) + 1}" + project_rows.append(work_item_id) + self.rows[work_item_id] = SimpleNamespace( + id=work_item_id, + name=data.name, + type_id=data.type_id, + completed_at=None, + archived_at=None, + ) + return self.rows[work_item_id] + + def retrieve(self, *, project_id, work_item_id, **kwargs): + row = self.rows[work_item_id] + project_rows = self.project_ids[str(project_id)] + if ( + self.mark_second_non_bug + and str(project_id).startswith("second-") + and work_item_id in project_rows[-self.mark_second_non_bug :] + ): + return SimpleNamespace(**{**vars(row), "type_id": "not-bug"}) + return row + + def seeded(run_id: str, *, mark_second_non_bug: int = 0): + run8 = run_id[:8] + main_id = f"main-{run8}" + main_name = f"EVAL {run8}" + work_items = WorkItems(mark_second_non_bug=mark_second_non_bug) + plane = SimpleNamespace( + projects=Projects(main_id, main_name), + work_items=work_items, + ) + ctx = { + "run_id": run_id, + "run8": run8, + "task_id": "R6", + "project_id": main_id, + "project_name": main_name, + "items": {}, + "item_ids": [], + "bug_type": {"id": "bug-1", "name": "Bug"}, + "bug_type_workspace_level": False, + "randomized_truth": {}, + } + seed_second_project(plane, "ws", ctx) + return plane, ctx + + first_plane, first = seeded("22222222cccccccc") # deterministic intended counts 3 / 2 + second_plane, second = seeded("aabbccdd11223344", mark_second_non_bug=2) # intended 4 / 5 + assert "second_project_ids" not in first + assert "second_project_ids" not in second + + first_intended = first["randomized_truth"]["R6.open_bug_counts"] + second_truth = second["randomized_truth"]["R6.open_bug_counts"] + assert (first_intended["intended_main"], first_intended["intended_second"]) != ( + second_truth["intended_main"], + second_truth["intended_second"], + ) + assert (first["r6_main_bug_count"], first["r6_second_bug_count"]) == (3, 2) + assert (second["r6_main_bug_count"], second["r6_second_bug_count"]) == (4, 3) + # Intended counts say B wins 5-to-4. API readback says main wins 4-to-3; + # the verifier must use the API-confirmed seed oracle. + assert second["r6_more_bugs_project"] == second["project_name"] + assert second_truth["confirmed"]["winner"] == second["project_name"] + + run = { + "final_text": f"project: {second['project_name']}", + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], } - lines = seed_plan(groups) - blob = "\n".join(lines) - for g in groups: - assert ( - g.split("_")[0] in blob - or g in blob - or g.replace("_", " ") in blob - or any(g in line for line in lines) - ), f"seed_plan missing {g}: {lines}" - # Specific fixtures named - assert "Sprint 12" in blob - assert "Checkout revamp" in blob - assert "1.2.0" in blob - assert "Acme Corp" in blob - - def test_seed_plan_empty_needs_only_project(): - lines = seed_plan(set()) - assert any("project" in line for line in lines) - # project line + default workspace customers enable note - assert any("customers" in line for line in lines) - assert len(lines) == 2 - - test_seed_plan_covers_all_groups() - test_seed_plan_empty_needs_only_project() - - def test_seed_module_ast_has_all_group_handlers(): - src = inspect.getsource(seed_mod.seed) - for group in ( - "labels", - "items", - "bug_type", - "cycles", - "module", - "intake", - "customer", - "release", - "second_project", - ): - assert f'"{group}"' in src or f"'{group}'" in src, group - - def test_seed_enables_project_features_immediately_after_create(monkeypatch): - from types import SimpleNamespace - - from plane.models.projects import ProjectFeature, UpdateProject - from plane.models.workspaces import WorkspaceFeature - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - calls: list[tuple] = [] - - class _Projects: - def create(self, workspace_slug, data): - calls.append(("create", workspace_slug, getattr(data, "name", None))) - return SimpleNamespace(id="proj-main") - - def update(self, workspace_slug, project_id, data): - assert isinstance(data, UpdateProject) - calls.append(("update", project_id, data.model_dump(exclude_none=True))) - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - assert isinstance(data, ProjectFeature) - calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) - return data - - class _Workspaces: - def update_features(self, workspace_slug, data): - assert isinstance(data, WorkspaceFeature) - calls.append(("ws_update_features", data.model_dump(exclude_none=True))) - return data - - plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) - ctx: dict = {} - seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx) - - assert ctx["project_id"] == "proj-main" - kinds = [c[0] for c in calls] - assert kinds == ["create", "ws_update_features", "update", "update_features"] - # Workspace customers enabled for C1 preconditions - assert calls[1][1].get("customers") is True - assert "work_item_types" not in calls[1][1] - # Project enable calls target the created id - assert calls[2][1] == "proj-main" - assert calls[3][1] == "proj-main" - upd = calls[2][2] - assert upd.get("cycle_view") is True - assert upd.get("is_time_tracking_enabled") is True - feat = calls[3][2] - assert feat.get("cycles") is True - - def test_seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): - from types import SimpleNamespace - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - calls: list[tuple] = [] - - class _Projects: - def create(self, workspace_slug, data): - return SimpleNamespace(id="proj-s5") - - def update(self, workspace_slug, project_id, data): - calls.append(("update", data.model_dump(exclude_none=True))) - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - calls.append(("features", data.model_dump(exclude_none=True))) - return data - - class _Workspaces: - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {"customers": True}) - - def update_features(self, workspace_slug, data): - calls.append(("ws_features", data.model_dump(exclude_none=True))) - return data - - plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) - ctx: dict = {} - seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) - assert ctx["feature_exclude"] == ["cycles", "worklogs"] - assert ctx["ws_feature_exclude"] == ["customers"] - assert ctx["s5_left_customers_off"] is True - # Excluded features are written OFF, not omitted. The workspace outlives the run, so - # omitting the write leaves the previous rep's value and S5's precondition never holds. - ws = next(c[1] for c in calls if c[0] == "ws_features") - assert ws.get("customers") is False - assert ctx["workspace_features_prior"] == {"customers": True} - upd = next(c[1] for c in calls if c[0] == "update") - assert upd.get("cycle_view") is False - assert upd.get("is_time_tracking_enabled") is False - assert upd.get("module_view") is True - feat = next(c[1] for c in calls if c[0] == "features") - assert feat.get("cycles") is False - assert feat.get("modules") is True - - def test_seed_cycles_create_add_then_backdate(monkeypatch): - from types import SimpleNamespace - - from plane.models.cycles import CreateCycle, UpdateCycle - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - calls: list[tuple] = [] - cycle_seq = {"n": 0} - - class _Cycles: - def create(self, workspace_slug, project_id, data): - assert isinstance(data, CreateCycle) - cycle_seq["n"] += 1 - cid = f"cyc-{cycle_seq['n']}" - calls.append( - ( - "create", - { - "name": data.name, - "start_date": data.start_date, - "end_date": data.end_date, - "id": cid, - }, - ) - ) - return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) + ], + "call_source": "test", + "evidence_trace_available": True, + } + ok, note = asyncio.run(verify_r6(second_plane, second, run)) + assert ok is True, note + wrong_run = {**run, "final_text": f"project: {second['second_project_name']}"} + wrong_ok, wrong_note = asyncio.run(verify_r6(second_plane, second, wrong_run)) + assert wrong_ok is False, wrong_note + + +def test_baseline_snapshot_failure_surfaces_before_workspace_mutation(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + + def fail_list(**kwargs): + raise RuntimeError("customers unreadable") + + plane = SimpleNamespace( + projects=SimpleNamespace(create=lambda **kwargs: SimpleNamespace(id="project-1", identifier="EVDEADBEEF")), + customers=SimpleNamespace( + list=fail_list, + properties=SimpleNamespace(list=lambda **kwargs: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kwargs: _Page([]))), + ) + context: dict[str, Any] = {} - def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): - calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) + with pytest.raises(RuntimeError, match="workspace baseline snapshot: list customers failed"): + seed_mod.seed(plane, "deadbeefcafebabe", set(), context, task_id="W10") - def update(self, workspace_slug, project_id, cycle_id, data): - assert isinstance(data, UpdateCycle) - calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) - return SimpleNamespace(id=cycle_id, end_date=data.end_date) + assert context["project_id"] == "project-1" - class _Projects: - def create(self, workspace_slug, data): - return SimpleNamespace(id="proj-1") - def update(self, workspace_slug, project_id, data): - return SimpleNamespace(id=project_id) +def test_workspace_feature_snapshot_failure_prevents_mutation(): + updates: list[Any] = [] + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("feature read failed")), + update_features=lambda **kwargs: updates.append(kwargs), + ) + ) - def update_features(self, workspace_slug, project_id, data): - return data + with pytest.raises(RuntimeError, match="workspace feature snapshot failed before mutation"): + seed_mod.enable_workspace_features(plane, "ws") - class _Workspaces: - def update_features(self, workspace_slug, data): - return data + assert updates == [] - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {}) - class _Users: - def get_me(self): - return SimpleNamespace(id="user-1") +@pytest.mark.parametrize( + "context_key,context_value", + [("second_project_id", "second-1"), ("second_project_ids", ["second-1"])], +) +def test_teardown_deletes_second_project_from_current_or_legacy_context_key(context_key, context_value): + deleted: list[str] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kwargs: _Page([]), + properties=SimpleNamespace(list=lambda **kwargs: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kwargs: _Page([]))), + projects=SimpleNamespace(delete=lambda **kwargs: deleted.append(str(kwargs["project_id"]))), + work_item_types=SimpleNamespace(list=lambda **kwargs: []), + ) + context = { + "workspace_slug": "ws", + "project_id": "main-1", + context_key: context_value, + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + + seed_mod.teardown(plane, context) + + assert deleted == ["second-1", "main-1"] + + +def _seed_plan_covers_all_groups(_monkeypatch): + groups = { + "items", + "labels", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + } + lines = seed_plan(groups) + blob = "\n".join(lines) + for g in groups: + assert ( + g.split("_")[0] in blob or g in blob or g.replace("_", " ") in blob or any(g in line for line in lines) + ), f"seed_plan missing {g}: {lines}" + # Specific fixtures named + assert "Sprint 12" in blob + assert "Checkout revamp" in blob + assert "1.2.0" in blob + assert "Acme Corp" in blob + + +def _seed_plan_empty_needs_only_project(_monkeypatch): + lines = seed_plan(set()) + assert any("project" in line for line in lines) + # project line + default workspace customers enable note + assert any("customers" in line for line in lines) + assert len(lines) == 2 + + +def _seed_module_ast_has_all_group_handlers(_monkeypatch): + src = inspect.getsource(seed_mod.seed) + for group in ( + "labels", + "items", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + ): + assert f'"{group}"' in src or f"'{group}'" in src, group + + +def _seed_enables_project_features_immediately_after_create(monkeypatch): + from types import SimpleNamespace - class _States: - def list(self, workspace_slug, project_id): - return SimpleNamespace( - results=[ - SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), - SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), - ] - ) + from plane.models.projects import ProjectFeature, UpdateProject + from plane.models.workspaces import WorkspaceFeature - item_n = {"n": 0} + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) - class _WorkItems: - def create(self, workspace_slug, project_id, data): - item_n["n"] += 1 - return SimpleNamespace( - id=f"wi-{item_n['n']}", - name=data.name, - state="st-started", - created_at="2026-01-01", + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + calls.append(("create", workspace_slug, getattr(data, "name", None))) + return SimpleNamespace(id="proj-main") + + def update(self, workspace_slug, project_id, data): + assert isinstance(data, UpdateProject) + calls.append(("update", project_id, data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + assert isinstance(data, ProjectFeature) + calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": False}) + + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(("ws_update_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="tag-unrelated", version=L3_TAG_VERSION)]), + delete=lambda **kw: None, + ) + ), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx, task_id="W10") + + assert ctx["project_id"] == "proj-main" + kinds = [c[0] for c in calls] + assert kinds == ["create", "ws_update_features", "update", "update_features"] + # Workspace customers enabled for C1 preconditions + assert calls[1][1].get("customers") is True + assert "work_item_types" not in calls[1][1] + # Project enable calls target the created id + assert calls[2][1] == "proj-main" + assert calls[3][1] == "proj-main" + upd = calls[2][2] + assert upd.get("cycle_view") is True + assert upd.get("is_time_tracking_enabled") is True + feat = calls[3][2] + assert feat.get("cycles") is True + assert ctx["workspace_baseline"] == { + "customers": set(), + "release_tags": {"tag-unrelated"}, + "customer_properties": set(), + "work_item_types": None, + "work_item_properties": None, + } + + +def _seed_collision_skips_before_create(monkeypatch): + from evals.tasks.skip import TaskSkipped + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + creates: list[Any] = [] + plane = SimpleNamespace( + projects=SimpleNamespace(create=lambda **kw: creates.append(kw)), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace( + tags=SimpleNamespace(list=lambda **kw: _Page([SimpleNamespace(id="tag-collision", version=L3_TAG_VERSION)])) + ), + ) + ctx: dict[str, Any] = {} + + with pytest.raises(TaskSkipped, match=r"^env:fixture-collision:release_tags:eval-rc1"): + seed_mod.seed(plane, run_id="collision123456", needs=set(), ctx=ctx, task_id="L3") + + assert creates == [] + assert ctx["project_id"] is None + assert ctx["workspace_baseline"] == { + "customers": None, + "release_tags": None, + "customer_properties": None, + "work_item_types": None, + "work_item_properties": None, + } + + +def _seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-s5") + + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": True}) + + def update_features(self, workspace_slug, data): + calls.append(("ws_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) + assert ctx["feature_exclude"] == ["cycles", "worklogs"] + assert ctx["ws_feature_exclude"] == ["customers"] + assert ctx["s5_left_customers_off"] is True + # Excluded features are written OFF, not omitted. The workspace outlives the run, so + # omitting the write leaves the previous rep's value and S5's precondition never holds. + ws = next(c[1] for c in calls if c[0] == "ws_features") + assert ws.get("customers") is False + assert ctx["workspace_features_prior"] == {"customers": True} + upd = next(c[1] for c in calls if c[0] == "update") + assert upd.get("cycle_view") is False + assert upd.get("is_time_tracking_enabled") is False + assert upd.get("module_view") is True + feat = next(c[1] for c in calls if c[0] == "features") + assert feat.get("cycles") is False + assert feat.get("modules") is True + + +def _seed_cycles_create_add_then_backdate(monkeypatch): + from types import SimpleNamespace + + from plane.models.cycles import CreateCycle, UpdateCycle + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + cycle_seq = {"n": 0} + + class _Cycles: + def create(self, workspace_slug, project_id, data): + assert isinstance(data, CreateCycle) + cycle_seq["n"] += 1 + cid = f"cyc-{cycle_seq['n']}" + calls.append( + ( + "create", + { + "name": data.name, + "start_date": data.start_date, + "end_date": data.end_date, + "id": cid, + }, ) + ) + return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) - def update(self, workspace_slug, project_id, work_item_id, data): - return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) + def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): + calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) - class comments: - @staticmethod - def create(**kw): - return SimpleNamespace(id="c1") + def update(self, workspace_slug, project_id, cycle_id, data): + assert isinstance(data, UpdateCycle) + calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) + return SimpleNamespace(id=cycle_id, end_date=data.end_date) - plane = SimpleNamespace( - projects=_Projects(), - workspaces=_Workspaces(), - cycles=_Cycles(), - users=_Users(), - states=_States(), - work_items=_WorkItems(), - ) - ctx: dict = {} - seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) - - # Filter to Sprint-12-related create/add/update sequence (first cycle is past). - past_id = ctx["cycle_past_id"] - # Must create both cycles before any backdate update of past. - create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] - assert len(create_idxs) == 2 - past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) - # Created with temporary *future* end_date (active), not the final past end. - assert past_create[1]["end_date"] > past_create[1]["start_date"] - # At least one add to past cycle before its update - past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] - past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] - assert past_adds, "expected add_work_items on Sprint 12" - assert past_updates, "expected backdate update on Sprint 12" - assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" - # Backdated end matches W6 seed ctx; differs from create-time active end - backdated_end = calls[past_updates[0]][1]["end_date"] - assert ctx["cycle_past_seed_end_date"] == backdated_end - assert backdated_end != past_create[1]["end_date"] - assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] - # Active cycle: create with future end; never backdated - cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) - assert cur_create[1]["end_date"] - cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] - assert cur_updates == [] - - def test_seed_enables_features_on_second_project_too(monkeypatch): - from types import SimpleNamespace - - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") - monkeypatch.delenv("REDIS_HOST", raising=False) - monkeypatch.delenv("REDIS_PORT", raising=False) - - creates: list[str] = [] - enables: list[str] = [] - - class _Projects: - def create(self, workspace_slug, data): - pid = f"p-{len(creates)}" - creates.append(pid) - return SimpleNamespace(id=pid) - - def update(self, workspace_slug, project_id, data): - enables.append(("update", project_id)) - return SimpleNamespace(id=project_id) - - def update_features(self, workspace_slug, project_id, data): - enables.append(("features", project_id)) - return data - - # Minimal stubs so second_project seed gets past bug_type + work items. - class _Workspaces: - def get_features(self, workspace_slug): - return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) - - def update_features(self, workspace_slug, data): - enables.append(("ws_features", workspace_slug)) - return data - - class _WorkItemTypes: - def list(self, **kw): - return [SimpleNamespace(id="bug-1", name="Bug")] - - def create(self, **kw): - return SimpleNamespace(id="bug-1", name="Bug") - - def import_to_project(self, **kw): - return None - - class _WorkItems: - def create(self, **kw): - return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-1") - plane = SimpleNamespace( - projects=_Projects(), - workspaces=_Workspaces(), - work_item_types=_WorkItemTypes(), - work_items=_WorkItems(), - ) - ctx: dict = {} - # second_project path also seeds bug_type when missing - seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) - - assert len(creates) == 2 - # Each create followed by update + update_features for that project id - assert ("update", creates[0]) in enables - assert ("features", creates[0]) in enables - assert ("update", creates[1]) in enables - assert ("features", creates[1]) in enables - - test_seed_plan_behaviours() - test_seed_module_ast_has_all_group_handlers() - with pytest.MonkeyPatch.context() as mp: - test_seed_enables_project_features_immediately_after_create(mp) - with pytest.MonkeyPatch.context() as mp: - test_seed_s5_leaves_cycles_worklogs_and_customers_off(mp) - with pytest.MonkeyPatch.context() as mp: - test_seed_cycles_create_add_then_backdate(mp) - with pytest.MonkeyPatch.context() as mp: - test_seed_enables_features_on_second_project_too(mp) + def update(self, workspace_slug, project_id, data): + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + return data + + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {}) + + class _Users: + def get_me(self): + return SimpleNamespace(id="user-1") + + class _States: + def list(self, workspace_slug, project_id): + return SimpleNamespace( + results=[ + SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), + ] + ) + + item_n = {"n": 0} + + class _WorkItems: + def create(self, workspace_slug, project_id, data): + item_n["n"] += 1 + return SimpleNamespace( + id=f"wi-{item_n['n']}", + name=data.name, + state="st-started", + created_at="2026-01-01", + ) + + def update(self, workspace_slug, project_id, work_item_id, data): + return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) + + class comments: + @staticmethod + def create(**kw): + return SimpleNamespace(id="c1") + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + cycles=_Cycles(), + users=_Users(), + states=_States(), + work_items=_WorkItems(), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) + + # Filter to Sprint-12-related create/add/update sequence (first cycle is past). + past_id = ctx["cycle_past_id"] + # Must create both cycles before any backdate update of past. + create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] + assert len(create_idxs) == 2 + past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) + # Created with temporary *future* end_date (active), not the final past end. + assert past_create[1]["end_date"] > past_create[1]["start_date"] + # At least one add to past cycle before its update + past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] + past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] + assert past_adds, "expected add_work_items on Sprint 12" + assert past_updates, "expected backdate update on Sprint 12" + assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" + # Backdated end matches W6 seed ctx; differs from create-time active end + backdated_end = calls[past_updates[0]][1]["end_date"] + assert ctx["cycle_past_seed_end_date"] == backdated_end + assert backdated_end != past_create[1]["end_date"] + assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] + # Active cycle: create with future end; never backdated + cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) + assert cur_create[1]["end_date"] + cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] + assert cur_updates == [] + + +def _seed_enables_features_on_second_project_too(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + creates: list[str] = [] + enables: list[str] = [] + + class _Projects: + def create(self, workspace_slug, data): + pid = f"p-{len(creates)}" + creates.append(pid) + return SimpleNamespace(id=pid) + + def update(self, workspace_slug, project_id, data): + enables.append(("update", project_id)) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + enables.append(("features", project_id)) + return data + + # Minimal stubs so second_project seed gets past bug_type + work items. + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) + + def update_features(self, workspace_slug, data): + enables.append(("ws_features", workspace_slug)) + return data + + class _WorkItemTypes: + def list(self, **kw): + return [SimpleNamespace(id="bug-1", name="Bug")] + + def create(self, **kw): + return SimpleNamespace(id="bug-1", name="Bug") + + def import_to_project(self, **kw): + return None + + class _WorkItems: + def create(self, **kw): + return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + work_item_types=_WorkItemTypes(), + work_items=_WorkItems(), + ) + ctx: dict = {} + # second_project path also seeds bug_type when missing + seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) + + assert len(creates) == 2 + # Each create followed by update + update_features for that project id + assert ("update", creates[0]) in enables + assert ("features", creates[0]) in enables + assert ("update", creates[1]) in enables + assert ("features", creates[1]) in enables + + +_SEED_CASES = case_params( + _seed_plan_covers_all_groups, + _seed_plan_empty_needs_only_project, + _seed_module_ast_has_all_group_handlers, + _seed_enables_project_features_immediately_after_create, + _seed_collision_skips_before_create, + _seed_s5_leaves_cycles_worklogs_and_customers_off, + _seed_cycles_create_add_then_backdate, + _seed_enables_features_on_second_project_too, +) + + +@pytest.mark.parametrize("case", _SEED_CASES) +def test_seed_behaviours(case, monkeypatch): + case(monkeypatch) def test_excluding_pages_turns_page_view_off_despite_its_true_default(monkeypatch): @@ -471,7 +715,15 @@ def update_features(self, workspace_slug, data): calls.append(data.model_dump(exclude_none=True)) return data - plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) + plane = SimpleNamespace( + workspaces=_Workspaces(), + projects=SimpleNamespace(delete=lambda **k: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + ) seed_mod.teardown( plane, { @@ -483,228 +735,705 @@ def update_features(self, workspace_slug, data): assert calls and calls[0].get("customers") is prior -def test_teardown_behaviours(): - def test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown(): - from types import SimpleNamespace - - calls: list = [] +def _teardown_leaves_workspace_alone_when_prior_unknown(): + from types import SimpleNamespace - class _Workspaces: - def update_features(self, workspace_slug, data): - calls.append(data) - return data + calls: list = [] - plane = SimpleNamespace(workspaces=_Workspaces(), projects=SimpleNamespace(delete=lambda **k: None)) - seed_mod.teardown( - plane, - { - "workspace_slug": "test-ws", - "workspace_features_prior": {"customers": None}, - "project_id": None, - }, - ) - assert calls == [] + class _Workspaces: + def update_features(self, workspace_slug, data): + calls.append(data) + return data - def test_teardown_deletes_release_tag_and_customer_property(): - from evals.seed import teardown + plane = SimpleNamespace( + workspaces=_Workspaces(), + projects=SimpleNamespace(delete=lambda **k: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + ) + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": None}, + "project_id": None, + }, + ) + assert calls == [] - plane = _TeardownPlane() - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "project_name": "EVAL x", - "workspace_objects": [ - {"kind": "release_tag", "id": "tag-tracked"}, - {"kind": "customer_property", "id": "prop-tracked"}, - ], - } - teardown(plane, ctx) - kinds = {k for k, _ in plane.deleted} - assert "release_tag" in kinds - assert "customer_property" in kinds - # Tracked ids deleted - assert ("release_tag", "tag-tracked") in plane.deleted - assert ("customer_property", "prop-tracked") in plane.deleted - test_teardown_leaves_workspace_alone_when_the_prior_value_is_unknown() - test_teardown_deletes_release_tag_and_customer_property() +def _teardown_deletes_release_tag_and_customer_property(): + from evals.seed import teardown + plane = _TeardownPlane() + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "project_name": "EVAL x", + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + "workspace_objects": [ + {"kind": "release_tag", "id": "tag-tracked"}, + {"kind": "customer_property", "id": "prop-tracked"}, + ], + } + teardown(plane, ctx) + kinds = {k for k, _ in plane.deleted} + assert "release_tag" in kinds + assert "customer_property" in kinds + # Tracked ids deleted + assert ("release_tag", "tag-tracked") in plane.deleted + assert ("customer_property", "prop-tracked") in plane.deleted + + +@pytest.mark.parametrize( + "case", + case_params( + _teardown_leaves_workspace_alone_when_prior_unknown, + _teardown_deletes_release_tag_and_customer_property, + ), +) +def test_teardown_behaviours(case): + case() -def test_preclean_behaviours(): - def test_preclean_removes_stale_tag_and_property(): - from evals.seed import _preclean_ws3_workspace_artifacts - deleted: list[tuple[str, str]] = [] +def test_teardown_aggregates_failures_after_attempting_every_object(): + from evals.seed import TeardownError, teardown - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-old", version=L3_TAG_VERSION)]), - delete=lambda **kw: deleted.append(("tag", kw["tag_id"])), - ) - ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="p-old", display_name=L4_PROP_DISPLAY, name="x")]), - delete=lambda **kw: deleted.append(("prop", kw["property_id"])), - ) - ) + delete_calls: list[tuple[str, str]] = [] - _preclean_ws3_workspace_artifacts(Plane(), "ws") - assert ("tag", "t-old") in deleted - assert ("prop", "p-old") in deleted + def fail_delete(kind: str, object_id: str) -> None: + delete_calls.append((kind, object_id)) + raise RuntimeError(f"cannot delete {kind} {object_id}") - def test_preclean_delete_failure_raises_for_infra_seed(): - from evals.seed import _preclean_ws3_workspace_artifacts + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + delete=lambda **kw: fail_delete("customer", kw["customer_id"]), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + delete=lambda **kw: fail_delete("release", kw["release_id"]), + tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + projects=SimpleNamespace(delete=lambda **kw: fail_delete("project", kw["project_id"])), + work_item_types=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + ) + context = { + "workspace_slug": "ws", + "project_id": "project-main", + "project_name": "EVAL cleanup", + "second_project_ids": ["project-second"], + "workspace_objects": [ + {"kind": "customer", "id": "customer-1"}, + {"kind": "release", "id": "release-1"}, + ], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + + with pytest.raises(TeardownError) as caught: + teardown(plane, context) + + assert delete_calls == [ + ("customer", "customer-1"), + ("release", "release-1"), + ("project", "project-second"), + ("project", "project-main"), + ] + assert len(caught.value.failures) == 4 + assert {failure.target for failure in caught.value.failures} == { + "customer-1", + "release-1", + "project-second", + "EVAL cleanup", + } + + +def test_teardown_customer_baseline_behaviours(capsys): + cases = ( + { + "name": "pre-existing name match", + "customer_id": "customer-existing", + "baseline": {"customer-existing"}, + "tracked": False, + "deleted": False, + "warns": False, + }, + { + "name": "agent-created name match", + "customer_id": "customer-agent", + "baseline": set(), + "tracked": False, + "deleted": True, + "warns": False, + }, + { + "name": "tracked id wins over baseline", + "customer_id": "customer-tracked", + "baseline": {"customer-tracked"}, + "tracked": True, + "deleted": True, + "warns": False, + }, + { + "name": "unavailable baseline fails closed", + "customer_id": "customer-unknown", + "baseline": None, + "tracked": False, + "deleted": False, + "warns": True, + }, + ) - class Plane: - releases = SimpleNamespace( - tags=SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="t-stuck", version=L3_TAG_VERSION)]), - delete=lambda **kw: (_ for _ in ()).throw(RuntimeError("403 forbidden")), - ) - ) - customers = SimpleNamespace( - properties=SimpleNamespace( - list=lambda **kw: _Page([]), - delete=lambda **kw: None, - ) + for case in cases: + with pytest.MonkeyPatch.context() as mp: + mp.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + deleted: list[str] = [] + customer = SimpleNamespace(id=case["customer_id"], name="Acme Corp") + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda customer=customer, **kw: _Page([customer]), + delete=lambda deleted=deleted, **kw: deleted.append(str(kw["customer_id"])), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)), + projects=SimpleNamespace(delete=lambda **kw: None), ) + workspace_objects = [{"kind": "customer", "id": case["customer_id"]}] if case["tracked"] else [] + context = { + "workspace_slug": "test-ws", + "project_id": None, + "workspace_objects": workspace_objects, + "workspace_baseline": { + "customers": case["baseline"], + "release_tags": set(), + "customer_properties": set(), + }, + } + if case["warns"]: + with pytest.raises(seed_mod.TeardownError) as caught: + seed_mod.teardown(plane, context) + assert "baseline unavailable" in str(caught.value) + else: + seed_mod.teardown(plane, context) + output = capsys.readouterr().out + assert (case["customer_id"] in deleted) is case["deleted"], case["name"] + assert ("customers baseline unavailable" in output) is case["warns"], case["name"] + if case["warns"]: + assert case["customer_id"] in output + + +def test_preexisting_workspace_bug_is_reused_and_not_deleted(): + deleted_types: list[str] = [] + imported_types: list[str] = [] + bug = SimpleNamespace(id="bug-existing", name="Bug") + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [bug], + create=lambda **kw: pytest.fail("pre-existing Bug must be reused"), + delete=lambda **kw: deleted_types.append(str(kw["type_id"])), + properties=SimpleNamespace(list=lambda **kw: []), + ), + workspace_work_item_properties=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + work_item_types=SimpleNamespace( + import_to_project=lambda **kw: imported_types.extend(str(value) for value in kw["work_item_type_ids"]), + ), + work_item_properties=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S1", + "workspace_slug": "ws", + "project_id": "project-1", + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": {"bug-existing"}, + "work_item_properties": set(), + }, + } - with pytest.raises(RuntimeError, match="preclean|failed to delete|eval-rc1|release tag"): - _preclean_ws3_workspace_artifacts(Plane(), "ws") + seed_mod.seed_item_type(plane, "ws", context) + seed_mod.teardown(plane, context) - def test_preclean_empty_list_is_silent(): - from evals.seed import _preclean_ws3_workspace_artifacts + assert imported_types == ["bug-existing"] + assert context["bug_type_created"] is False + assert context["workspace_objects"] == [] + assert deleted_types == [] - class Plane: - releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)) - customers = SimpleNamespace( - properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None) - ) - _preclean_ws3_workspace_artifacts(Plane(), "ws") # no raise +@pytest.mark.parametrize( + ("baseline", "should_delete"), + [ + pytest.param({"severity-existing"}, False, id="pre-existing-preserved"), + pytest.param(set(), True, id="agent-created-deleted"), + ], +) +def test_teardown_severity_property_uses_seed_baseline(baseline, should_delete): + deleted: list[str] = [] + bug = SimpleNamespace(id="bug-existing", name="Bug") + severity = SimpleNamespace(id="severity-existing", display_name="Severity") + plane = SimpleNamespace( + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [bug], + properties=SimpleNamespace(list=lambda **kw: [severity.id]), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda **kw: [severity], + delete=lambda **kw: deleted.append(str(kw["property_id"])), + ), + work_item_properties=SimpleNamespace(list=lambda **kw: [severity], delete=lambda **kw: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S1", + "workspace_slug": "ws", + "project_id": "project-1", + "bug_type": {"id": bug.id, "name": bug.name}, + "bug_type_workspace_level": True, + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": {bug.id}, + "work_item_properties": baseline, + }, + } - test_preclean_removes_stale_tag_and_property() - test_preclean_delete_failure_raises_for_infra_seed() - test_preclean_empty_list_is_silent() + seed_mod.teardown(plane, context) + assert (severity.id in deleted) is should_delete -def test_l2_activity_behaviours(): - def test_l2_activity_gate_raises_when_empty(): - from types import SimpleNamespace - from evals.seed import R5_TITLE, _gate_activity_worker - from evals.tasks.skip import TaskSkipped +@pytest.mark.parametrize( + ("baseline", "should_delete"), + [ + pytest.param({"incident-existing"}, False, id="pre-existing-preserved"), + pytest.param(set(), True, id="agent-created-deleted"), + ], +) +def test_teardown_workspace_incident_uses_seed_baseline(baseline, should_delete): + deleted: list[str] = [] + incident = SimpleNamespace(id="incident-existing", name="Incident") + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [incident], + delete=lambda **kw: deleted.append(str(kw["type_id"])), + ), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S3", + "workspace_slug": "ws", + "project_id": "project-1", + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": baseline, + "work_item_properties": None, + }, + } - class Plane: - work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[]))) + seed_mod.teardown(plane, context) - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - with pytest.raises(TaskSkipped, match="env:no-activity-worker"): - _gate_activity_worker(Plane(), "ws", ctx) + assert (incident.id in deleted) is should_delete - def test_l2_activity_gate_proceeds_when_nonempty(): - from types import SimpleNamespace - from evals.seed import R5_TITLE, _gate_activity_worker +def test_preclean_behaviours(): + from evals.seed import check_workspace_fixture_collisions + from evals.tasks.skip import TaskSkipped - class Plane: - work_items = SimpleNamespace( - activities=SimpleNamespace(list=lambda **kw: SimpleNamespace(results=[SimpleNamespace(id="a1")])) + cases = ( + { + "name": "release tag collision", + "customers": [], + "tags": [SimpleNamespace(id="tag-old", version=L3_TAG_VERSION)], + "properties": [], + "category": "release_tags", + "fixture_name": L3_TAG_VERSION, + "checked_categories": {"release_tags"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "customer property collision", + "customers": [], + "tags": [], + "properties": [SimpleNamespace(id="prop-old", display_name=L4_PROP_DISPLAY, name="x")], + "category": "customer_properties", + "fixture_name": L4_PROP_DISPLAY, + "checked_categories": {"customer_properties"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "customer collision", + "customers": [SimpleNamespace(id="customer-old", name="Acme")], + "tags": [], + "properties": [], + "category": "customers", + "fixture_name": "Acme Corp", + "checked_categories": {"customers"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "Bug Severity collision", + "customers": [], + "tags": [], + "properties": [], + "workspace_types": [SimpleNamespace(id="type-bug", name="Bug")], + "workspace_properties": [SimpleNamespace(id="severity-old", display_name="Severity")], + "category": "work_item_properties", + "fixture_name": "Severity", + "checked_categories": {"work_item_properties"}, + }, + { + "name": "Incident collision", + "customers": [], + "tags": [], + "properties": [], + "workspace_types": [SimpleNamespace(id="incident-old", name="Incident")], + "workspace_properties": [], + "category": "work_item_types", + "fixture_name": "Incident", + "checked_categories": {"work_item_types"}, + }, + { + "name": "clean workspace", + "customers": [SimpleNamespace(id="customer-other", name="Other Corp")], + "tags": [SimpleNamespace(id="tag-other", version="v2")], + "properties": [SimpleNamespace(id="prop-other", display_name="Region", name="region")], + "category": None, + "fixture_name": None, + "checked_categories": {"customers", "release_tags", "customer_properties"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "release tag irrelevant to checked category", + "customers": [], + "tags": [SimpleNamespace(id="tag-unrelated", version=L3_TAG_VERSION)], + "properties": [], + "category": None, + "fixture_name": None, + "checked_categories": {"customers"}, + "workspace_types": [SimpleNamespace(id="incident-unrelated", name="Incident")], + "workspace_properties": [], + }, + ) + + for case in cases: + with pytest.MonkeyPatch.context(): + deleted: list[tuple[str, str]] = [] + customers = case["customers"] + tags = case["tags"] + properties = case["properties"] + workspace_types = case["workspace_types"] + workspace_properties = case["workspace_properties"] + plane = SimpleNamespace( + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda tags=tags, **kw: _Page(tags), + delete=lambda deleted=deleted, **kw: deleted.append(("tag", kw["tag_id"])), + ) + ), + customers=SimpleNamespace( + list=lambda customers=customers, **kw: _Page(customers), + delete=lambda deleted=deleted, **kw: deleted.append(("customer", kw["customer_id"])), + properties=SimpleNamespace( + list=lambda properties=properties, **kw: _Page(properties), + delete=lambda deleted=deleted, **kw: deleted.append(("property", kw["property_id"])), + ), + ), + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda workspace_types=workspace_types, **kw: workspace_types, + properties=SimpleNamespace( + list=lambda workspace_properties=workspace_properties, **kw: ( + [row.id for row in workspace_properties] if kw.get("type_id") == "type-bug" else [] + ) + ), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda workspace_properties=workspace_properties, **kw: workspace_properties + ), ) - ctx = {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}} - _gate_activity_worker(Plane(), "ws", ctx) # no raise + if case["category"] is None: + check_workspace_fixture_collisions(plane, "ws", case["checked_categories"]) + else: + expected = f"env:fixture-collision:{case['category']}:{case['fixture_name']}" + with pytest.raises(TaskSkipped) as caught: + check_workspace_fixture_collisions(plane, "ws", case["checked_categories"]) + assert caught.value.reason.startswith(expected), case["name"] + assert case["fixture_name"] in caught.value.reason + assert "python -m evals.cleanup --sentinels --yes" in caught.value.reason + assert deleted == [], case["name"] + + +def test_collision_category_coverage_matches_task_prompts(): + from evals.seed import ( + CUSTOMER_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + EVALUATION_RELEASE_TAG_VERSION, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + collision_categories, + ) + from evals.tasks.catalog import TASKS + + prompt_categories = ( + (CUSTOMER_NAME, "customers"), + (EVALUATION_RELEASE_TAG_VERSION, "release_tags"), + (EVALUATION_CUSTOMER_PROPERTY_NAME, "customer_properties"), + (SEVERITY_PROPERTY_NAME, "work_item_properties"), + (INCIDENT_TYPE_NAME, "work_item_types"), + ) + for task in TASKS: + task_id = str(task["id"]) + categories = collision_categories(set(task.get("needs") or set()), task_id) + prompt = str(task.get("prompt") or "") + for fixture_name, category in prompt_categories: + if fixture_name in prompt: + assert category in categories, f"task {task_id} prompt references {fixture_name!r}; missing {category}" + + +@pytest.mark.parametrize( + ("context", "read_result", "expected_error", "match"), + [ + pytest.param( + {"project_id": "p1", "items": {}}, + [], + RuntimeError, + "fixture error: missing work_item_id", + id="missing-work-item-is-fixture-error", + ), + pytest.param( + {"project_id": None, "items": {R5_TITLE: "wi-r5"}}, + [], + RuntimeError, + "fixture error: missing project_id", + id="missing-project-is-fixture-error", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + ConnectionError("activity backend unavailable"), + ConnectionError, + "activity backend unavailable", + id="read-failure-propagates-as-infrastructure", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + [], + TaskSkipped, + "^env:no-activity-worker$", + id="successful-empty-read-is-capability-skip", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + [SimpleNamespace(id="a1")], + None, + None, + id="successful-nonempty-read-proceeds", + ), + pytest.param( + { + "task_id": "L2", + "project_id": "p1", + "items": {R5_TITLE: "wi-r5"}, + "l2_comment_phrases": ["hidden seeded comment"], + }, + [SimpleNamespace(id="a1", comment="unrelated activity")], + RuntimeError, + "fixture error: activity readback omitted randomized comment evidence", + id="nonempty-read-without-seeded-evidence-is-fixture-error", + ), + ], +) +def test_l2_activity_gate_outcomes(context, read_result, expected_error, match): + from evals.seed import _gate_activity_worker + + def list_activities(**kwargs): + if isinstance(read_result, BaseException): + raise read_result + return SimpleNamespace(results=read_result) - test_l2_activity_gate_raises_when_empty() - test_l2_activity_gate_proceeds_when_nonempty() + plane = SimpleNamespace(work_items=SimpleNamespace(activities=SimpleNamespace(list=list_activities))) + if expected_error is None: + _gate_activity_worker(plane, "ws", context) + else: + with pytest.raises(expected_error, match=match): + _gate_activity_worker(plane, "ws", context) -def test_create_behaviours(monkeypatch): - def test_create_project_retries_409_then_succeeds(monkeypatch): - attempts: list[str] = [] +def _create_project_retries_409_then_succeeds(monkeypatch): + attempts: list[str] = [] - class FakeProjects: - def create(self, *, workspace_slug, data): - ident = data.identifier - attempts.append(ident) - if len(attempts) < 3: - raise HttpError("Project identifier already taken", 409) - return MagicMock(id="proj-ok", identifier=ident) + class FakeProjects: + def create(self, *, workspace_slug, data): + ident = data.identifier + attempts.append(ident) + if len(attempts) < 3: + raise HttpError("Project identifier already taken", 409) + return MagicMock(id="proj-ok", identifier=ident) - plane = MagicMock() - plane.projects = FakeProjects() + plane = MagicMock() + plane.projects = FakeProjects() - # Force deterministic retries after first collision. - suffixes = iter(["AAAA", "BBBB"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + # Force deterministic retries after first collision. + suffixes = iter(["AAAAAAAA", "BBBBBBBB"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - project = create_project_with_identifier_retry( + project = create_project_with_identifier_retry( + plane, + "ws", + name="EVAL abcd", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + ) + assert project.id == "proj-ok" + assert attempts[0] == "EVDEADBEEF" + assert len(attempts) == 3 + assert attempts[1] != attempts[0] + assert attempts[2] != attempts[1] + assert attempts[1] == "EVAAAAAAAA" + assert attempts[2] == "EVBBBBBBBB" + + +def _create_project_raises_after_max_409s(monkeypatch): + attempts: list[str] = [] + + class Always409: + def create(self, *, workspace_slug, data): + attempts.append(data.identifier) + raise HttpError("identifier already taken", 409) + + plane = MagicMock() + plane.projects = Always409() + suffixes = iter( + [ + "11111111", + "22222222", + "33333333", + "44444444", + "55555555", + "66666666", + "77777777", + "should-not-use", + ] + ) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( plane, "ws", - name="EVAL abcd", + name="EVAL x", identifier_prefix="EV", - initial_suffix="DEAD", + initial_suffix="00000000", ) - assert project.id == "proj-ok" - assert attempts[0] == "EVDEAD" - assert len(attempts) == 3 - assert attempts[1] != attempts[0] - assert attempts[2] != attempts[1] - assert attempts[1] == "EVAAAA" - assert attempts[2] == "EVBBBB" - - def test_create_project_raises_after_max_409s(monkeypatch): - attempts: list[str] = [] - - class Always409: - def create(self, *, workspace_slug, data): - attempts.append(data.identifier) - raise HttpError("identifier already taken", 409) - - plane = MagicMock() - plane.projects = Always409() - suffixes = iter(["1111", "2222", "3333", "should-not-use"]) - monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( - plane, - "ws", - name="EVAL x", - identifier_prefix="EV", - initial_suffix="0000", - ) - assert ei.value.status_code == 409 - assert len(attempts) == 3 - assert attempts[0] == "EV0000" - assert attempts[1] != attempts[0] - assert attempts[1] == "EV1111" - assert attempts[2] == "EV2222" - - def test_create_project_non_collision_error_does_not_retry(): - class Fail500: - def create(self, *, workspace_slug, data): - raise HttpError("server error", 500) - - plane = MagicMock() - plane.projects = Fail500() - with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( - plane, - "ws", - name="EVAL x", - identifier_prefix="EV", - initial_suffix="0000", - ) - assert ei.value.status_code == 500 + assert ei.value.status_code == 409 + assert len(attempts) == 8 + assert attempts[0] == "EV00000000" + assert attempts[1] != attempts[0] + assert attempts[1] == "EV11111111" + assert attempts[-1] == "EV77777777" + + +def _create_project_non_collision_error_does_not_retry(_monkeypatch): + class Fail500: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + plane = MagicMock() + plane.projects = Fail500() + with pytest.raises(HttpError) as ei: + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="00000000", + ) + assert ei.value.status_code == 500 - with pytest.MonkeyPatch.context() as mp: - test_create_project_retries_409_then_succeeds(mp) - with pytest.MonkeyPatch.context() as mp: - test_create_project_raises_after_max_409s(mp) - test_create_project_non_collision_error_does_not_retry() + +def _identifier_stays_within_plane_limit(_monkeypatch): + plane = SimpleNamespace( + projects=SimpleNamespace( + create=lambda **kwargs: SimpleNamespace(id="project", identifier=kwargs["data"].identifier) + ) + ) + project = create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="12345678", + ) + assert project.identifier == "EV12345678" + assert len(project.identifier) <= seed_mod.PLANE_PROJECT_IDENTIFIER_MAX_LENGTH + + with pytest.raises(ValueError, match="12-character limit"): + create_project_with_identifier_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="TOO-LONG", + initial_suffix="12345678", + ) + + +@pytest.mark.parametrize( + "case", + case_params( + _create_project_retries_409_then_succeeds, + _create_project_raises_after_max_409s, + _create_project_non_collision_error_does_not_retry, + _identifier_stays_within_plane_limit, + ), +) +def test_create_behaviours(case, monkeypatch): + case(monkeypatch) def test_identifier_collision_requires_status_and_language(): @@ -715,114 +1444,203 @@ def test_identifier_collision_requires_status_and_language(): assert is_identifier_collision(HttpError("identifier already taken", 500)) is False -def test_cleanup_behaviours(monkeypatch, capsys): - def test_cleanup_dry_run_never_calls_delete(monkeypatch, capsys): - projects = [ - SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), - SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), - SimpleNamespace(id="p3", name="Production", identifier="PROD"), - ] - delete_calls: list[Any] = [] +def _cleanup_dry_run_never_calls_delete(monkeypatch, capsys, _yes): + projects = [ + SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), + SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), + SimpleNamespace(id="p3", name="Production", identifier="PROD"), + ] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + rc = cleanup_mod.main([]) # dry-run + assert rc == 0 + assert delete_calls == [] + out = capsys.readouterr().out + assert "EVAL deadbeef" in out + assert "dry-run" in out + assert "Production" not in out # prefix filter + + +def _cleanup_yes_deletes(monkeypatch, capsys, _yes): + projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + rc = cleanup_mod.main(["--yes"]) + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0]["project_id"] == "p1" + + +def _cleanup_sentinel_mode(monkeypatch, capsys, yes): + delete_calls: list[tuple[str, str]] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="customer-eval", name="Acme Corp"), + SimpleNamespace(id="customer-short", name="Acme"), + SimpleNamespace(id="customer-other", name="Other Corp"), + ] + ), + delete=lambda **kw: delete_calls.append(("customer", kw["customer_id"])), + properties=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="property-eval", display_name="Eval Industry", name="eval-industry"), + SimpleNamespace(id="property-other", display_name="Region", name="region"), + ] + ), + delete=lambda **kw: delete_calls.append(("customer_property", kw["property_id"])), + ), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="tag-eval", version="eval-rc1"), + SimpleNamespace(id="tag-other", version="v2"), + ] + ), + delete=lambda **kw: delete_calls.append(("release_tag", kw["tag_id"])), + ) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="type-bug", name="Bug"), + SimpleNamespace(id="type-incident", name="Incident"), + SimpleNamespace(id="type-epic", name="Epic"), + ], + properties=SimpleNamespace(list=lambda **kw: ["severity-eval"] if kw.get("type_id") == "type-bug" else []), + delete=lambda **kw: delete_calls.append(("work_item_type", kw["type_id"])), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="severity-eval", display_name="Severity"), + SimpleNamespace(id="property-region", display_name="Region"), + ], + delete=lambda **kw: delete_calls.append(("work_item_property", kw["property_id"])), + ), + ) + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + args = ["--sentinels", "--yes"] if yes else ["--sentinels"] + assert cleanup_mod.main(args) == 0 + output = capsys.readouterr().out + for fixture_name in ("Acme Corp", "eval-rc1", "Eval Industry", "Incident", "Severity"): + assert fixture_name in output + assert "Other Corp" not in output + assert "Region" not in output + if yes: + assert set(delete_calls) == { + ("customer", "customer-eval"), + ("customer", "customer-short"), + ("release_tag", "tag-eval"), + ("customer_property", "property-eval"), + ("work_item_type", "type-incident"), + ("work_item_property", "severity-eval"), + } + assert "deleted sentinel" in output + assert "would delete sentinel" not in output + else: + assert delete_calls == [] + assert output.count("would delete sentinel") == 6 + assert "dry-run" in output + + +@pytest.mark.parametrize( + ("case", "yes"), + [ + pytest.param(_cleanup_dry_run_never_calls_delete, None, id="project-dry-run"), + pytest.param(_cleanup_yes_deletes, None, id="project-delete"), + pytest.param(_cleanup_sentinel_mode, False, id="sentinel-dry-run"), + pytest.param(_cleanup_sentinel_mode, True, id="sentinel-delete"), + ], +) +def test_cleanup_behaviours(case, yes, monkeypatch, capsys): + case(monkeypatch, capsys, yes) + + +def _list_projects_with_prefix_filters(): + projects = [ + SimpleNamespace(id="1", name="EVAL a"), + SimpleNamespace(id="2", name="Other"), + SimpleNamespace(id="3", name="EVAL b"), + SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " + ] + calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + calls.append({"workspace_slug": workspace_slug, "params": params}) + assert params is not None + assert params.per_page == 100 + # SDK always populates next_cursor even on last page. + return SimpleNamespace( + results=projects, + next_page_results=False, + next_cursor="100:0:0", + ) - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "3"] + assert len(calls) == 1 # one page only — no infinite loop on next_cursor + assert calls[0]["params"].cursor is None - def delete(self, **kwargs): - delete_calls.append(kwargs) - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) +def _list_projects_two_page_pagination(): + page1 = [SimpleNamespace(id="1", name="EVAL one")] + page2 = [SimpleNamespace(id="2", name="EVAL two")] + seen_cursors: list[Any] = [] - rc = cleanup_mod.main([]) # dry-run - assert rc == 0 - assert delete_calls == [] - out = capsys.readouterr().out - assert "EVAL deadbeef" in out - assert "dry-run" in out - assert "Production" not in out # prefix filter - - def test_cleanup_yes_deletes(monkeypatch, capsys): - projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] - delete_calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") - - def delete(self, **kwargs): - delete_calls.append(kwargs) - - plane = MagicMock() - plane.projects = FakeProjects() - monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) - rc = cleanup_mod.main(["--yes"]) - assert rc == 0 - assert len(delete_calls) == 1 - assert delete_calls[0]["project_id"] == "p1" - - with pytest.MonkeyPatch.context() as mp: - test_cleanup_dry_run_never_calls_delete(mp, capsys) - with pytest.MonkeyPatch.context() as mp: - test_cleanup_yes_deletes(mp, capsys) - - -def test_list_projects_behaviours(): - def test_list_projects_with_prefix_filters(): - projects = [ - SimpleNamespace(id="1", name="EVAL a"), - SimpleNamespace(id="2", name="Other"), - SimpleNamespace(id="3", name="EVAL b"), - SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " - ] - calls: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - calls.append({"workspace_slug": workspace_slug, "params": params}) - assert params is not None - assert params.per_page == 100 - # SDK always populates next_cursor even on last page. + class FakeProjects: + def list(self, workspace_slug=None, params=None): + seen_cursors.append(getattr(params, "cursor", None)) + if params.cursor is None: return SimpleNamespace( - results=projects, - next_page_results=False, + results=page1, + next_page_results=True, next_cursor="100:0:0", ) + assert params.cursor == "100:0:0" + return SimpleNamespace( + results=page2, + next_page_results=False, + next_cursor="200:0:0", + ) - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "3"] - assert len(calls) == 1 # one page only — no infinite loop on next_cursor - assert calls[0]["params"].cursor is None - - def test_list_projects_two_page_pagination(): - page1 = [SimpleNamespace(id="1", name="EVAL one")] - page2 = [SimpleNamespace(id="2", name="EVAL two")] - seen_cursors: list[Any] = [] - - class FakeProjects: - def list(self, workspace_slug=None, params=None): - seen_cursors.append(getattr(params, "cursor", None)) - if params.cursor is None: - return SimpleNamespace( - results=page1, - next_page_results=True, - next_cursor="100:0:0", - ) - assert params.cursor == "100:0:0" - return SimpleNamespace( - results=page2, - next_page_results=False, - next_cursor="200:0:0", - ) + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "2"] + assert seen_cursors == [None, "100:0:0"] - plane = MagicMock() - plane.projects = FakeProjects() - got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") - assert [p.id for p in got] == ["1", "2"] - assert seen_cursors == [None, "100:0:0"] - test_list_projects_with_prefix_filters() - test_list_projects_two_page_pagination() +@pytest.mark.parametrize( + "case", + case_params(_list_projects_with_prefix_filters, _list_projects_two_page_pagination), +) +def test_list_projects_behaviours(case): + case() diff --git a/tests/fixtures/evals_historical_rows.jsonl b/tests/fixtures/evals_schema_v0_rows.jsonl similarity index 100% rename from tests/fixtures/evals_historical_rows.jsonl rename to tests/fixtures/evals_schema_v0_rows.jsonl From b4378c0f632604ffdb01118eb84445ff7631f31f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 18:32:28 +0530 Subject: [PATCH 33/93] Stop verifiers passing tasks they could not verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit found only 9 of 35 verifiers unqualified sound. Four defect classes, each swept rather than patched at the reported site. R7 returned True unconditionally when the agent emitted "transition: unrestricted" — the API was queried and the result never consulted. It is rewritten to compare reported state names and groups against live state, with no branch returning True without an API comparison. Oracles are captured at seed time and checked for tampering. C2 previously graded against a repo constant; grading against post-agent live state instead would let the agent write its own answer, so the changelog is read back at seed time and verification requires live == baseline. Every verifier was audited for that shape. Read tasks now require evidence-bearing provenance: a successful response that exposed the seed-time fact for the target entity, matched at record time so no response body is persisted. Tool identity is never asserted — that coupling was removed deliberately and breaks on any server refactor. Verifier read failures raise instead of returning False, across 21 except-blocks with an explicit per-site decision, because a failed read by the verifier is infrastructure and not evidence the agent failed. A 404 for a resource the task asked the agent to create remains an agent failure. W2 requires exactly Done, W4 the exact label, W3 exact comment text; W8 and W9 stopped asking for properties the API cannot confirm. Five verifiers that read only the first page now paginate. Co-Authored-By: Claude Opus 5 (1M context) --- evals/evidence.py | 200 +++++ evals/state_oracle.py | 21 + evals/tasks/answers.py | 80 ++ evals/tasks/catalog.py | 36 +- evals/tasks/cross.py | 148 ++-- evals/tasks/debias.py | 255 +++--- evals/tasks/lookups.py | 31 +- evals/tasks/read.py | 236 +++--- evals/tasks/schema.py | 119 +-- evals/tasks/skip.py | 11 +- evals/tasks/verification.py | 24 + evals/tasks/write.py | 340 +++----- tests/evals/tasks/test_catalog.py | 240 +++--- tests/evals/tasks/test_debias_verifiers.py | 60 +- tests/evals/tasks/test_gate_recovery.py | 4 +- tests/evals/tasks/test_lookups.py | 23 + tests/evals/tasks/test_output_contracts.py | 390 ++++++++- tests/evals/tasks/test_pagination.py | 142 ++++ .../evals/tasks/test_verifier_read_errors.py | 510 ++++++++++++ tests/evals/tasks/test_verifiers.py | 786 ++++++++++-------- 20 files changed, 2455 insertions(+), 1201 deletions(-) create mode 100644 evals/evidence.py create mode 100644 evals/state_oracle.py create mode 100644 evals/tasks/verification.py create mode 100644 tests/evals/tasks/test_lookups.py create mode 100644 tests/evals/tasks/test_pagination.py create mode 100644 tests/evals/tasks/test_verifier_read_errors.py diff --git a/evals/evidence.py b/evals/evidence.py new file mode 100644 index 00000000..af3331c0 --- /dev/null +++ b/evals/evidence.py @@ -0,0 +1,200 @@ +"""Target-bound response-evidence matching without retaining response bodies. + +Seeders register hidden, per-run sentinel values under a non-sensitive label. Drivers +compare each Plane response with those values only when the request targets the seeded +entity, then retain only the labels that matched. CLI proxies consume the matching +configuration from a one-shot file before the agent starts; sentinel values never enter +agent-visible argv/config, result rows, or payload-free sidecars. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +EVIDENCE_SENTINELS_ENV = "EVAL_EVIDENCE_SENTINELS_JSON" +TARGET_ENTITY_EVIDENCE = "target-entity-hidden-fact" + + +def normalize_evidence_sentinels(value: Any) -> dict[str, tuple[str, ...]]: + """Return a validated label-to-sentinel mapping, dropping empty values.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[str, ...]] = {} + for raw_label, raw_values in value.items(): + label = str(raw_label or "").strip() + if not label: + continue + values: Sequence[Any] + if isinstance(raw_values, str): + values = (raw_values,) + elif isinstance(raw_values, Sequence): + values = raw_values + else: + continue + clean_values: list[str] = [] + for item in values: + if item is None: + continue + text = str(item).strip() + if text: + clean_values.append(text) + clean = tuple(dict.fromkeys(clean_values)) + if clean: + normalized[label] = clean + return normalized + + +def normalize_evidence_targets(value: Any) -> dict[str, tuple[str, ...]]: + """Return a validated label-to-target-ID mapping, dropping empty values.""" + return normalize_evidence_sentinels(value) + + +def configured_evidence_labels(sentinels: Any, targets: Any) -> tuple[str, ...]: + """Return labels that have both response values and target entity IDs.""" + values_by_label = normalize_evidence_sentinels(sentinels) + targets_by_label = normalize_evidence_targets(targets) + return tuple(sorted(values_by_label.keys() & targets_by_label.keys())) + + +def encode_evidence_sentinels(value: Any) -> str: + """Serialize a temporary driver/proxy configuration, never a result-row field.""" + normalized = normalize_evidence_sentinels(value) + return json.dumps(normalized, ensure_ascii=True, separators=(",", ":")) + + +def decode_evidence_sentinels(value: str | None) -> dict[str, tuple[str, ...]]: + """Decode a temporary driver/proxy configuration, failing closed on bad input.""" + if not value: + return {} + try: + raw = json.loads(value) + except (TypeError, ValueError): + return {} + return normalize_evidence_sentinels(raw) + + +def encode_evidence_config(sentinels: Any, targets: Any) -> str: + """Serialize the proxy-only matching configuration.""" + return json.dumps( + { + "sentinels": normalize_evidence_sentinels(sentinels), + "targets": normalize_evidence_targets(targets), + }, + ensure_ascii=True, + separators=(",", ":"), + ) + + +def decode_evidence_config(value: str | None) -> tuple[dict[str, tuple[str, ...]], dict[str, tuple[str, ...]]]: + """Decode proxy-only matching configuration, failing closed on malformed input.""" + if not value: + return {}, {} + try: + raw = json.loads(value) + except (TypeError, ValueError): + return {}, {} + if not isinstance(raw, Mapping): + return {}, {} + return ( + normalize_evidence_sentinels(raw.get("sentinels")), + normalize_evidence_targets(raw.get("targets")), + ) + + +def write_evidence_config(path: Path, sentinels: Any, targets: Any) -> None: + """Create a private, one-shot proxy configuration outside the agent cwd.""" + payload = encode_evidence_config(sentinels, targets) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(payload) + + +def consume_evidence_config(path: Path | None) -> tuple[dict[str, tuple[str, ...]], dict[str, tuple[str, ...]]]: + """Read and unlink a one-shot proxy configuration, failing closed.""" + if path is None: + return {}, {} + try: + raw = path.read_text(encoding="utf-8") + except OSError: + return {}, {} + finally: + try: + path.unlink() + except OSError: + pass + return decode_evidence_config(raw) + + +def _request_targets(request_args: Any, target_ids: Sequence[str]) -> bool: + targets = set(target_ids) + + def contains(value: Any) -> bool: + if isinstance(value, Mapping): + return any(contains(item) for item in value.values()) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return any(contains(item) for item in value) + return value is not None and str(value) in targets + + return contains(request_args) + + +def observed_sentinel_labels( + response_text: str, + sentinels: Any, + *, + request_args: Any, + evidence_targets: Any, +) -> list[str]: + """Return labels whose target request exposed a hidden value in its response.""" + text = str(response_text or "") + if not text: + return [] + normalized = normalize_evidence_sentinels(sentinels) + targets = normalize_evidence_targets(evidence_targets) + return sorted( + label + for label, values in normalized.items() + if label in targets + and _request_targets(request_args, targets[label]) + and any(value in text for value in values) + ) + + +def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, target_ids: Sequence[Any]) -> None: + """Register API-confirmed values and the entity IDs whose reads may prove them.""" + clean_values: list[str] = [] + for value in values: + if value is None: + continue + text = str(value).strip() + if text: + clean_values.append(text) + clean = tuple(dict.fromkeys(clean_values)) + if not clean: + raise RuntimeError("target evidence has no API-confirmed sentinel values") + clean_targets = tuple(dict.fromkeys(str(value).strip() for value in target_ids if str(value).strip())) + if not clean_targets: + raise RuntimeError("target evidence has no seeded target entity ids") + context["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: clean} + context["evidence_targets"] = {TARGET_ENTITY_EVIDENCE: clean_targets} + + +__all__ = [ + "EVIDENCE_SENTINELS_ENV", + "TARGET_ENTITY_EVIDENCE", + "configured_evidence_labels", + "consume_evidence_config", + "decode_evidence_config", + "decode_evidence_sentinels", + "encode_evidence_config", + "encode_evidence_sentinels", + "normalize_evidence_sentinels", + "normalize_evidence_targets", + "observed_sentinel_labels", + "set_target_evidence", + "write_evidence_config", +] diff --git a/evals/state_oracle.py b/evals/state_oracle.py new file mode 100644 index 00000000..dc1fca03 --- /dev/null +++ b/evals/state_oracle.py @@ -0,0 +1,21 @@ +"""Neutral project-state normalization shared by seeders and verifiers.""" + +from __future__ import annotations + +from typing import Any + + +def state_name_group_pairs(rows: list[Any]) -> list[str]: + """Return exact ``NAME | group: GROUP`` pairs, rejecting incomplete rows.""" + pairs: list[str] = [] + for state in rows: + name = str(getattr(state, "name", None) or "").strip() + raw_group = getattr(state, "group", None) + group = str(getattr(raw_group, "value", raw_group) or "").strip() + if not name or not group: + raise RuntimeError(f"project state lacks name or group: {state!r}") + pairs.append(f"{name} | group: {group}") + return pairs + + +__all__ = ["state_name_group_pairs"] diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py index 7868af55..d7a66389 100644 --- a/evals/tasks/answers.py +++ b/evals/tasks/answers.py @@ -4,8 +4,11 @@ import re from collections import Counter +from html import unescape from typing import Any +from evals.evidence import TARGET_ENTITY_EVIDENCE + def word_boundary(value: str) -> re.Pattern[str]: """Compile a case-insensitive word-boundary match for an exact seeded value.""" @@ -101,9 +104,86 @@ def get_final_text(run: dict[str, Any]) -> str: return run.get("final_text") or "" +def normalize_rich_text(value: Any) -> str: + """Return exact comparable text from a rich-text API model, mapping, or string. + + Prefer authoritative stripped fields when the API exposes them, then normalize HTML + entities, tags, and whitespace. Case and punctuation remain significant. + """ + + def field(name: str) -> Any: + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + candidates = ( + value if isinstance(value, str) else None, + field("comment_stripped"), + field("description_stripped"), + field("comment_html"), + field("description_html"), + ) + for candidate in candidates: + if isinstance(candidate, str) and candidate.strip(): + without_tags = re.sub(r"<[^>]*>", " ", candidate) + return " ".join(unescape(without_tags).split()) + return "" + + +def has_response_evidence(run: dict[str, Any], label: str = TARGET_ENTITY_EVIDENCE) -> bool: + """Return whether a successful Plane response exposed the target's hidden fact. + + Tool identity is deliberately irrelevant. The transport records only non-sensitive + sentinel labels after matching the in-memory response; no response body is required. + """ + calls = run.get("calls") + if not isinstance(calls, list): + return False + return any( + isinstance(call, dict) and not bool(call.get("is_error")) and label in (call.get("observed_sentinels") or []) + for call in calls + ) + + +def answer_with_provenance( + answer_correct: bool, + answer_note: str, + run: dict[str, Any], +) -> tuple[bool, str]: + """Combine answer correctness with route-agnostic response evidence. + + The two facts stay separate in the note. A successful unrelated call has no target + label and therefore cannot satisfy provenance. + """ + calls = run.get("calls") + source = str(run.get("call_source") or "unknown") + driver_notes = run.get("driver_notes") + trace_incomplete = isinstance(driver_notes, list) and any( + isinstance(note, str) and note.startswith("proxy_sidecar_incomplete") for note in driver_notes + ) + available = bool(run.get("evidence_trace_available")) + provenance = not trace_incomplete and has_response_evidence(run) + if trace_incomplete: + provenance_note = f"trace incomplete (source={source}; proxy sidecar was not authoritative)" + elif provenance: + provenance_note = f"observed target-entity response evidence (source={source})" + elif not available: + provenance_note = f"unavailable (source={source}; response-evidence matching was not active)" + elif isinstance(calls, list) and calls: + successful = sum(1 for call in calls if isinstance(call, dict) and not bool(call.get("is_error"))) + provenance_note = ( + f"missing (0 evidence-bearing of {successful} successful Plane calls; {len(calls)} total; source={source})" + ) + else: + provenance_note = f"missing (0 Plane calls observed; source={source})" + note = f"answer_correct={str(bool(answer_correct)).lower()} ({answer_note}); provenance={provenance_note}" + return bool(answer_correct) and provenance, note + + __all__ = [ "contract_values", + "answer_with_provenance", "get_final_text", + "has_response_evidence", + "normalize_rich_text", "reports_contract_int", "reports_contract_value", "reports_contract_values", diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 73496ae4..2a481816 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,28 +80,30 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 2 +CATALOG_REVISION = 7 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. -The hash below covers prompts and tool sets, not ``needs`` or verifier bodies, because -prompt drift is the signal it was built to catch. That leaves a hole: correcting a seeder -changes the question a task puts to the agent while the fingerprint keeps asserting the -results are comparable. Bumping this closes the hole by making the redefinition visible. - -Revision 1 covers batteries 6-8. Revision 2 is the workspace/project feature-exclusion -correction: excluding a feature now writes it ``False`` instead of omitting the write, so -S5 is graded on three conditions the agent must actually satisfy rather than two plus one -the workspace already happened to be in. +Revision 7 makes read tasks require response evidence bound to the target entity and gives +C2/R7 randomised, immutable seed-time oracles; pre-revision read results are not comparable. +Revision 6 stops W8 and W9 asking for unverifiable logged-date and batching properties; +it also tightens W3/W10 end-state contracts and paginates affected verifier reads. Revision +5 makes R1-R6, I2, L2, and L5 require observed successful Plane tool-call provenance and +randomises their hidden truth. Read results across either transition are not comparable. +Revision 4 rewrote R7 into an exact live state-and-group listing. Revision 3 removed +declared per-task tool sets and call floors and added fixture names. Fixture names being +covered means swapping a task's fixtures no longer needs a manual bump; changing a seeder's +behaviour under the same name still does. """ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: - """Stable short hash (SHA-256/12) of CATALOG_REVISION plus each task's id, prompt, - tool sets and optimal_calls. + """Stable short hash of the revision and each task's ID, prompt and fixture names. - Fixtures and verifier bodies are deliberately excluded — prompt drift is the signal — - so bump CATALOG_REVISION when they redefine the question. A --tasks subset hashes - differently from the full catalog. + Everything hashed is a fact about what the agent was asked — never an expectation + about how it should answer. Fixture *names* are covered, so swapping a task's + fixtures is caught mechanically; seeder and verifier *bodies* are not, which is the + hole CATALOG_REVISION exists to close by hand. A --tasks subset hashes differently + from the full catalog. """ src = list(TASKS if tasks is None else tasks) payload: list[dict[str, Any]] = [] @@ -110,9 +112,7 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: { "id": t.get("id"), "prompt": t.get("prompt"), - "optimal_tools": sorted(t.get("optimal_tools") or []), - "alternate_tools": sorted(t.get("alternate_tools") or []), - "optimal_calls": t.get("optimal_calls"), + "needs": sorted(t.get("needs") or []), } ) document = {"revision": CATALOG_REVISION, "tasks": payload} diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py index 1422b080..8f59b7f9 100644 --- a/evals/tasks/cross.py +++ b/evals/tasks/cross.py @@ -2,23 +2,23 @@ from __future__ import annotations -import re from typing import Any -from evals.seed import ( +from evals.changelog import changelog_items, normalize_changelog_text +from evals.fixtures import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, R1_TITLE, - RELEASE_CHANGELOG_TEXT, - RELEASE_NAME, ) from evals.tasks.answers import ( + answer_with_provenance, contract_values, get_final_text, reports_contract_value, reports_contract_values, ) -from evals.tasks.lookups import as_id, find_item_by_name, ids +from evals.tasks.lookups import as_id, collect_paginated, find_item_by_name, ids +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: @@ -37,8 +37,12 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if r1 is None: return False, f"R1 item {R1_TITLE!r} not found in project" - customers = plane.customers.list(workspace_slug=workspace_slug) - rows = customers.results if hasattr(customers, "results") else customers + rows = collect_paginated( + lambda cursor: plane.customers.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) # Exact name only — do not match arbitrary acme* customers. acme = next((c for c in (rows or []) if (c.name or "").strip() == CUSTOMER_NAME), None) if acme is None: @@ -48,8 +52,13 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if not ctx.get("customer"): ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": acme.id}) - reqs = plane.customers.requests.list(workspace_slug=workspace_slug, customer_id=acme.id) - rrows = reqs.results if hasattr(reqs, "results") else reqs + rrows = collect_paginated( + lambda cursor: plane.customers.requests.list( + workspace_slug=workspace_slug, + customer_id=acme.id, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) sso = next( (r for r in (rrows or []) if (r.name or "").strip() == CUSTOMER_REQUEST_NAME), None, @@ -62,8 +71,13 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup # Require the R1 work item among customer-linked work items. try: - wi = plane.customers.work_items.list(workspace_slug=workspace_slug, customer_id=acme.id) - wi_rows = list(wi.results if hasattr(wi, "results") else wi or []) + wi_rows = collect_paginated( + lambda cursor: plane.customers.work_items.list( + workspace_slug=workspace_slug, + customer_id=acme.id, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) linked_ids = ids(wi_rows) # Plain string ids also count. for row in wi_rows: @@ -84,35 +98,19 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append(f"R1 item {r1.id} linked") except Exception as exc: - ok = False - notes.append(f"list customer work items failed: {exc}") + raise_verifier_read_error("C1", f"listing work items linked to customer {acme.id}", exc) return ok, "; ".join(notes) C1_TASK: dict[str, Any] = { "id": "C1", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ( f"Create customer '{CUSTOMER_NAME}' (if it does not already exist), add a " f"request named '{CUSTOMER_REQUEST_NAME}', and link that request to the work " f"item '{R1_TITLE}' in project {{project}}." ), - "optimal_calls": 4, - "optimal_tools": { - "list_customers", - "create_customer", - "create_customer_request", - "list_work_items", - }, - "alternate_tools": { - "retrieve_customer", - "manage_customer_work_items", - "list_customer_requests", - "list_customer_work_items", - "search_work_items", - "list_projects", - }, # No pre-seeded customer — agent creates; items needed for link target. "needs": {"items"}, "verify": verify_c1, @@ -120,51 +118,93 @@ async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """C2: exact contract fields report the release and every changelog item.""" + """C2: report the immutable release baseline with target-response evidence.""" final_text = get_final_text(run) notes: list[str] = [] ok = True - if not reports_contract_value(final_text, "release", RELEASE_NAME): + release = ctx.get("release") or {} + expected_release = str( + ctx.get("release_name") + or (release.get("name") if isinstance(release, dict) else getattr(release, "name", None)) + or "" + ) + if not expected_release: + return answer_with_provenance(False, "fixture missing: seeded release name is unavailable", run) + if not reports_contract_value(final_text, "release", expected_release): ok = False - notes.append(f"release values={contract_values(final_text, 'release')!r}; want [{RELEASE_NAME!r}]") + notes.append(f"release values={contract_values(final_text, 'release')!r}; want [{expected_release!r}]") else: - notes.append(f"release={RELEASE_NAME!r}") - - changelog = ctx.get("release_changelog_text") or RELEASE_CHANGELOG_TEXT - markers = list(re.finditer(r"Changelog entry\s+[^:]+:\s*", changelog, flags=re.IGNORECASE)) - shipped: list[str] = [] - for index, marker in enumerate(markers): - end = markers[index + 1].start() if index + 1 < len(markers) else len(changelog) - item = changelog[marker.end() : end].strip().rstrip(".").strip() - if item: - shipped.append(item) + notes.append(f"release={expected_release!r}") + + release_id = release.get("id") if isinstance(release, dict) else getattr(release, "id", release) + if not release_id: + return answer_with_provenance(False, "fixture missing: seeded release id is unavailable", run) + baseline = ctx.get("release_changelog_text") + if not isinstance(baseline, str) or not baseline.strip(): + return answer_with_provenance(False, "fixture missing: seeded changelog baseline is empty", run) + try: + live_release = plane.releases.retrieve( + workspace_slug=ctx["workspace_slug"], + release_id=release_id, + ) + response = plane.releases.changelog.retrieve( + workspace_slug=ctx["workspace_slug"], + release_id=release_id, + ) + except Exception as exc: + if is_verifier_not_found(exc): + return answer_with_provenance( + False, + f"seeded release/changelog no longer exists at verification ({exc})", + run, + ) + raise_verifier_read_error("C2", f"reading release {release_id} and its changelog", exc) + live_release_name = str(getattr(live_release, "name", None) or "").strip() + if live_release_name != expected_release: + return answer_with_provenance( + False, + f"release name was mutated after seeding: live={live_release_name!r}; baseline={expected_release!r}", + run, + ) + live = normalize_changelog_text(response) + if live != baseline: + if not live: + mutation_note = "changelog was mutated after seeding: live changelog is empty" + else: + mutation_note = f"changelog was mutated after seeding: live={live!r}; baseline={baseline!r}" + return answer_with_provenance(False, mutation_note, run) + shipped = changelog_items(baseline) if not shipped: - return False, f"seeded changelog does not contain parseable entries: {changelog!r}" + return answer_with_provenance( + False, + f"fixture missing: seeded changelog baseline has no parseable entries: {baseline!r}", + run, + ) if not reports_contract_values(final_text, "shipped", shipped): ok = False notes.append(f"shipped values={contract_values(final_text, 'shipped')!r}; want {shipped!r}") else: notes.append(f"{len(shipped)} exact shipped items") - return ok, "; ".join(notes) + return answer_with_provenance(ok, "; ".join(notes), run) + + +def _bind_c2(ctx: dict[str, Any]) -> dict[str, str]: + release = ctx.get("release") or {} + name = ctx.get("release_name") or (release.get("name") if isinstance(release, dict) else None) + return {"release_name": str(name or "")} C2_TASK: dict[str, Any] = { "id": "C2", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( - f"What shipped in release {RELEASE_NAME}? Summarize the changelog in any prose " - f"you like, then provide these exact contract lines: 'release: {RELEASE_NAME}' " + "What shipped in release {release_name}? Summarize the changelog in any prose " + "you like, then provide these exact contract lines: 'release: {release_name}' " "and one 'shipped: ' line per changelog item. " "For each 'shipped:' value, copy only the text after the changelog entry label, " "without its sentence-ending punctuation." ), - "optimal_calls": 2, - "optimal_tools": {"list_releases", "get_release_changelog"}, - "alternate_tools": { - "retrieve_release", - "list_release_work_items", - "update_release_changelog", - }, + "prompt_bind": _bind_c2, "needs": {"release"}, "verify": verify_c2, } diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 9b9ed5c3..1f84fe98 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -6,7 +6,7 @@ from plane.models.query_params import RetrieveQueryParams -from evals.seed import ( +from evals.fixtures import ( CUSTOMER_NAME, CYCLE_CURRENT, DEBIAS_CUSTOMER_PROP_DISPLAY, @@ -18,13 +18,15 @@ W8_TITLE, ) from evals.tasks.answers import ( + answer_with_provenance, contract_values, get_final_text, reports_contract_int, reports_contract_value, reports_contract_values, ) -from evals.tasks.lookups import ids, state_name +from evals.tasks.lookups import collect_paginated, ids +from evals.tasks.verification import raise_verifier_read_error I1_TITLE = R1_TITLE @@ -101,56 +103,38 @@ async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup I1_TASK: dict[str, Any] = { "id": "I1", "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, + "tags": {"write", "id_in_hand", "debias"}, "prompt": ("In project {project}, update work item {work_item_id}: set its priority to high."), "prompt_bind": _bind_item_uuid(I1_TITLE), - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - "retrieve_work_item_by_identifier", - }, "needs": {"items"}, "verify": verify_i1, } async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """I2: final text reports the identifier-target state via ``state: NAME``.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(I2_TITLE) - if not wid: - return False, f"seed item {I2_TITLE!r} missing" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - name = state_name(plane, workspace_slug, project_id, detail.state) + """I2: final text reports the API-confirmed seed state with call provenance.""" + name = str(ctx.get("i2_state_name") or "") if not name: - return False, "target state name unresolved" + return answer_with_provenance(False, "API-confirmed target state missing from seed ctx", run) final_text = get_final_text(run) - if reports_contract_value(final_text, "state", name): - return True, f"final text reports state {name!r} via contract" - return False, f"state values={contract_values(final_text, 'state')!r}; want [{name!r}]" + answer_correct = reports_contract_value(final_text, "state", name) + answer_note = ( + f"final text reports state {name!r} via contract" + if answer_correct + else f"state values={contract_values(final_text, 'state')!r}; want [{name!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) I2_TASK: dict[str, Any] = { "id": "I2", "author": "post-hoc-debias", - "tags": {"read", "tier1", "id_in_hand", "debias"}, + "tags": {"read", "id_in_hand", "debias"}, "prompt": ( "In project {project}, what is the current state of work item " "{work_item_identifier}? Return exactly one line: 'state: '." ), "prompt_bind": _bind_item_identifier(I2_TITLE), - "optimal_calls": 1, - "optimal_tools": {"retrieve_work_item_by_identifier"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - "list_states", - }, "needs": {"items"}, "verify": verify_i2, } @@ -181,17 +165,9 @@ async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup I3_TASK: dict[str, Any] = { "id": "I3", "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, + "tags": {"write", "id_in_hand", "debias"}, "prompt": ("In project {project}, add work item {work_item_id} to cycle {cycle_id}."), "prompt_bind": _bind_i3, - "optimal_calls": 1, - "optimal_tools": {"manage_cycle_work_items"}, - "alternate_tools": { - "list_cycles", - "list_cycle_work_items", - "list_work_items", - "retrieve_cycle", - }, "needs": {"items", "cycles"}, "verify": verify_i3, } @@ -220,17 +196,9 @@ async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup I4_TASK: dict[str, Any] = { "id": "I4", "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, + "tags": {"write", "id_in_hand", "debias"}, "prompt": ("In project {project}, attach label {label_id} to work item {work_item_id}."), "prompt_bind": _bind_i4, - "optimal_calls": 1, - "optimal_tools": {"manage_work_item_label"}, - "alternate_tools": { - "update_work_item", - "list_labels", - "retrieve_work_item", - "list_work_items", - }, "needs": {"items", "labels"}, "verify": verify_i4, } @@ -253,41 +221,41 @@ async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup I5_TASK: dict[str, Any] = { "id": "I5", "author": "post-hoc-debias", - "tags": {"write", "tier1", "id_in_hand", "debias"}, + "tags": {"write", "id_in_hand", "debias"}, "prompt": ("In project {project}, set the priority of work item {work_item_id} to low."), "prompt_bind": _bind_item_uuid(I3_TITLE), # footer item; not high-traffic elsewhere - "optimal_calls": 1, - "optimal_tools": {"update_work_item"}, - "alternate_tools": { - "retrieve_work_item", - "list_work_items", - "search_work_items", - }, "needs": {"items"}, "verify": verify_i5, } async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L1: 90-minute log exists and exact contract lines report the API summary.""" + """L1: 90-minute log exists and reporting uses an immutable target-id oracle.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] wid = (ctx.get("items") or {}).get(L1_TITLE) if not wid: - return False, f"seed item {L1_TITLE!r} missing" + return answer_with_provenance(False, f"seed item {L1_TITLE!r} missing", run) + expected_summary_ids = [str(value) for value in (ctx.get("l1_expected_summary_ids") or [])] + if expected_summary_ids != [str(wid)]: + return answer_with_provenance( + False, + f"L1 fixture oracle mismatch: summary ids={expected_summary_ids!r}; target={wid!r}", + run, + ) # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). logs = plane.work_items.work_logs.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] if 90 not in durations: - return False, f"no 90-minute work log on target item {wid}; durations={durations}" + return answer_with_provenance(False, f"no 90-minute work log on target item {wid}; durations={durations}", run) try: summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) sum_rows = list(raw or []) except Exception as exc: - return False, f"project worklog summary failed: {exc}" + raise_verifier_read_error("L1", "reading the project worklog summary", exc) summary_ids: list[str] = [] for row in sum_rows: @@ -299,22 +267,44 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if item_id and item_id not in summary_ids: summary_ids.append(item_id) if str(wid) not in summary_ids: - return False, f"target item {wid} missing from project worklog summary ids={summary_ids!r}" + return answer_with_provenance( + False, + f"target item {wid} missing from project worklog summary ids={summary_ids!r}", + run, + ) + if sorted(summary_ids) != sorted(expected_summary_ids): + return answer_with_provenance( + False, + f"worklog summary was mutated beyond the seeded oracle: live={summary_ids!r}; " + f"expected={expected_summary_ids!r}", + run, + ) final_text = get_final_text(run) if not reports_contract_value(final_text, "logged-minutes", "90"): - return False, f"logged-minutes values={contract_values(final_text, 'logged-minutes')!r}; want ['90']" - if not reports_contract_values(final_text, "summary-work-item-id", summary_ids): - return False, ( - f"summary-work-item-id values={contract_values(final_text, 'summary-work-item-id')!r}; want {summary_ids!r}" + return answer_with_provenance( + False, + f"logged-minutes values={contract_values(final_text, 'logged-minutes')!r}; want ['90']", + run, ) - return True, f"90m log on {wid} + exact contract for {len(summary_ids)} summary row(s)" + if not reports_contract_values(final_text, "summary-work-item-id", expected_summary_ids): + return answer_with_provenance( + False, + f"summary-work-item-id values={contract_values(final_text, 'summary-work-item-id')!r}; " + f"want {expected_summary_ids!r}", + run, + ) + return answer_with_provenance( + True, + f"90m log on {wid} + exact contract for {len(expected_summary_ids)} immutable summary row(s)", + run, + ) L1_TASK: dict[str, Any] = { "id": "L1", "author": "post-hoc-debias", - "tags": {"write", "read", "tier1", "long_tail", "debias"}, + "tags": {"write", "read", "long_tail", "debias"}, "prompt": ( f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " f"'{L1_TITLE}', then report the project's worklog summary. End with exactly " @@ -322,58 +312,36 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "'summary-work-item-id: ' line for every row returned " "by the project worklog summary. Include no other lines with those prefixes." ), - "optimal_calls": 3, - "optimal_tools": {"list_work_items", "create_work_log", "get_project_worklog_summary"}, - "alternate_tools": { - "search_work_items", - "list_work_logs", - "retrieve_work_item", - "list_projects", - }, "needs": {"items"}, "verify": verify_l1, } async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L2: target has activities; final text reports the count via ``count: N`` contract.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(L2_TITLE) - if not wid: - return False, f"seed item {L2_TITLE!r} missing" - try: - page = plane.work_items.activities.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - except Exception as exc: - return False, f"activities.list failed: {exc}" - rows = page.results if hasattr(page, "results") else page - n = len(list(rows or [])) - if n < 1: - return False, "no activities on target (seed comments should create some)" + """L2: final text reports the API-confirmed seed activity count with provenance.""" + n = ctx.get("l2_activity_count") + if not isinstance(n, int) or n < 1: + return answer_with_provenance(False, "API-confirmed activity count missing from seed ctx", run) final_text = get_final_text(run) - if not reports_contract_int(final_text, n): - return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" - return True, f"final text reports activity count {n} via contract" + answer_correct = reports_contract_int(final_text, n) + answer_note = ( + f"final text reports activity count {n} via contract" + if answer_correct + else f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + ) + return answer_with_provenance(answer_correct, answer_note, run) L2_TASK: dict[str, Any] = { "id": "L2", "author": "post-hoc-debias", - "tags": {"read", "tier1", "long_tail", "debias"}, + "tags": {"read", "long_tail", "debias"}, "prompt": ( f"In project {{project}}, list the activity history for the work item titled " f"'{L2_TITLE}'. Summarize how many activities there are and mention any " "notable comment phrases you see. End your answer with a line of the form " "'count: N' where N is the number of activities." ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_activities"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "list_work_item_comments", - "retrieve_work_item_activity", - }, "needs": {"items", "activity_feed"}, "verify": verify_l2, } @@ -383,10 +351,14 @@ async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup """L3: workspace has a release tag with version eval-rc1.""" workspace_slug = ctx["workspace_slug"] try: - page = plane.releases.tags.list(workspace_slug=workspace_slug) + rows = collect_paginated( + lambda cursor: plane.releases.tags.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) except Exception as exc: - return False, f"list release tags failed: {exc}" - rows = page.results if hasattr(page, "results") else page + raise_verifier_read_error("L3", "listing workspace release tags", exc) versions = {(getattr(t, "version", None) or "").strip() for t in (rows or [])} if L3_TAG_VERSION in versions: # Track for teardown if id available. @@ -405,16 +377,8 @@ async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup L3_TASK: dict[str, Any] = { "id": "L3", "author": "post-hoc-debias", - "tags": {"write", "tier1", "long_tail", "debias"}, + "tags": {"write", "long_tail", "debias"}, "prompt": (f"Create a release tag with version '{L3_TAG_VERSION}' (a version marker for the eval run)."), - "optimal_calls": 1, - "optimal_tools": {"create_release_tag"}, - "alternate_tools": { - "list_release_tags", - "retrieve_release_tag", - "list_releases", - "update_release_tag", - }, "needs": set(), # workspace-level tag; no project fixture required "verify": verify_l3, } @@ -445,10 +409,14 @@ async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if not customer_id: return False, "customer missing from seed" try: - props = plane.customers.properties.list(workspace_slug=workspace_slug) + prop_rows = collect_paginated( + lambda cursor: plane.customers.properties.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) except Exception as exc: - return False, f"list customer properties failed: {exc}" - prop_rows = props.results if hasattr(props, "results") else props + raise_verifier_read_error("L4", "listing workspace customer properties", exc) target_prop: Any | None = None for p in prop_rows or []: # Exact display_name match only (case-insensitive full match) — not substring "Industry". @@ -471,7 +439,7 @@ async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup try: values = plane.customers.property_values.list(workspace_slug=workspace_slug, customer_id=customer_id) except Exception as exc: - return False, f"get property values failed: {exc}" + raise_verifier_read_error("L4", f"reading property values for customer {customer_id}", exc) if not isinstance(values, dict): return False, f"unexpected property_values shape: {type(values)}" vals = values.get(pid) or values.get(str(pid)) or [] @@ -484,63 +452,40 @@ async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup L4_TASK: dict[str, Any] = { "id": "L4", "author": "post-hoc-debias", - "tags": {"write", "tier1", "long_tail", "debias"}, + "tags": {"write", "long_tail", "debias"}, "prompt": ( f"For customer '{CUSTOMER_NAME}', ensure there is a text customer property " f"named '{L4_PROP_DISPLAY}' and set its value to '{L4_PROP_VALUE}'." ), - "optimal_calls": 3, - "optimal_tools": { - "list_customers", - "create_customer_property", - "set_customer_property_values", - }, - "alternate_tools": { - "list_customer_properties", - "get_customer_property_values", - "retrieve_customer", - "update_customer_property", - }, "needs": {"customer"}, "verify": verify_l4, } async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """L5: final text reports the attachment count via ``count: N`` contract.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - wid = (ctx.get("items") or {}).get(L5_TITLE) - if not wid: - return False, f"seed item {L5_TITLE!r} missing" - try: - page = plane.work_items.attachments.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) - except Exception as exc: - return False, f"attachments.list failed: {exc}" - rows = page.results if hasattr(page, "results") else page - n = len(list(rows or [])) + """L5: final text reports the API-confirmed seed attachment count with provenance.""" + n = ctx.get("l5_attachment_count") + if not isinstance(n, int): + return answer_with_provenance(False, "API-confirmed attachment count missing from seed ctx", run) final_text = get_final_text(run) - if not reports_contract_int(final_text, n): - return False, f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" - return True, f"final text reports attachment count {n} via contract" + answer_correct = reports_contract_int(final_text, n) + answer_note = ( + f"final text reports attachment count {n} via contract" + if answer_correct + else f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + ) + return answer_with_provenance(answer_correct, answer_note, run) L5_TASK: dict[str, Any] = { "id": "L5", "author": "post-hoc-debias", - "tags": {"read", "tier1", "long_tail", "debias"}, + "tags": {"read", "long_tail", "debias"}, "prompt": ( f"In project {{project}}, how many file attachments does the work item titled " f"'{L5_TITLE}' have? End your answer with a line of the form 'count: N' " "where N is the number of file attachments." ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_attachments"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "get_work_item_attachment_download_url", - }, "needs": {"items"}, "verify": verify_l5, } diff --git a/evals/tasks/lookups.py b/evals/tasks/lookups.py index 8b658041..c5139f1c 100644 --- a/evals/tasks/lookups.py +++ b/evals/tasks/lookups.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from typing import Any from plane.errors.errors import HttpError @@ -25,6 +26,32 @@ def ids(items: Any) -> set[str]: return out +def collect_paginated(fetch_page: Callable[[str | None], Any]) -> list[Any]: + """Collect every result from a cursor-paginated SDK endpoint. + + ``fetch_page`` receives ``None`` for the first request and the prior response's + ``next_cursor`` thereafter. Endpoints documented as unpaginated may return a bare + list; in that case the list is already complete. + """ + rows: list[Any] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + while True: + page = fetch_page(cursor) + if isinstance(page, list): + rows.extend(page) + return rows + results = page.results if hasattr(page, "results") else page + rows.extend(list(results or [])) + if not bool(getattr(page, "next_page_results", False)): + return rows + next_cursor = str(getattr(page, "next_cursor", None) or "") + if not next_cursor or next_cursor in seen_cursors: + raise RuntimeError("paginated API reported another page without a new next_cursor") + seen_cursors.add(next_cursor) + cursor = next_cursor + + def find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: """Return all work items with exact name, newest first (by created_at).""" matches: list[Any] = [] @@ -89,7 +116,8 @@ def state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any if not state_id: return None page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - for s in page.results or []: + results = page.results if hasattr(page, "results") else page + for s in results or []: if str(s.id) == str(state_id): return getattr(s, "group", None) return None @@ -123,6 +151,7 @@ def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: __all__ = [ "as_id", + "collect_paginated", "count_open_urgent", "find_item_by_name", "find_items_by_name", diff --git a/evals/tasks/read.py b/evals/tasks/read.py index cc44ece6..8ddb0c83 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -2,91 +2,71 @@ from __future__ import annotations +from collections import Counter from typing import Any -from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, R5_TITLE +from evals.fixtures import R1_TITLE, R5_TITLE +from evals.state_oracle import state_name_group_pairs from evals.tasks.answers import ( + answer_with_provenance, contract_values, get_final_text, reports_contract_int, reports_contract_value, reports_contract_values, ) -from evals.tasks.lookups import count_open_urgent, find_item_by_name, state_name async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R1: final text must report the API-resolved state via ``state: NAME``.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - title = R1_TITLE - item = find_item_by_name(plane, workspace_slug, project_id, title) - if item is None: - return False, f"seeded item {title!r} not found" - detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - expected = state_name(plane, workspace_slug, project_id, detail.state) - if not expected: - # Prefer the seeded name when API is sparse. - expected = ctx.get("r1_state_name") + """R1: final text reports the API-confirmed seed state with call provenance.""" + expected = str(ctx.get("r1_state_name") or "") if not expected: - return False, "could not resolve expected state name from API" + return answer_with_provenance(False, "API-confirmed seed state missing", run) final_text = get_final_text(run) - if not reports_contract_value(final_text, "state", expected): - return False, f"final text must contain exactly 'state: {expected}'" - return True, f"final text reports state {expected!r} via contract" + answer_correct = reports_contract_value(final_text, "state", expected) + answer_note = ( + f"final text reports state {expected!r} via contract" + if answer_correct + else f"state values={contract_values(final_text, 'state')!r}; want [{expected!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) R1_TASK: dict[str, Any] = { "id": "R1", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( "In project {project}, what is the current state of the work item titled " f"'{R1_TITLE}'? Return exactly one line: 'state: '." ), - "optimal_calls": 1, - "optimal_tools": {"list_work_items"}, - "alternate_tools": { - "search_work_items", - "list_archived_work_items", - "count_work_items", - "retrieve_work_item", - "retrieve_work_item_by_identifier", - "list_projects", - "list_states", - }, "needs": {"items"}, "verify": verify_r1, } async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R2: final text reports the urgent-open count via ``count: N``.""" - workspace_slug = ctx["workspace_slug"] - project_id = ctx["project_id"] - expected = count_open_urgent(plane, workspace_slug, project_id) + """R2: final text reports the API-confirmed seed count with call provenance.""" + expected = ctx.get("r2_urgent_open_count") + if not isinstance(expected, int): + return answer_with_provenance(False, "API-confirmed urgent-open seed count missing", run) final_text = get_final_text(run) - if not reports_contract_int(final_text, expected): - return False, f"final text missing contract count: {expected} (need 'count: {expected}')" - return True, f"final text reports urgent-open count {expected} via contract" + answer_correct = reports_contract_int(final_text, expected) + answer_note = ( + f"final text reports urgent-open count {expected} via contract" + if answer_correct + else f"final text missing contract count: {expected} (need 'count: {expected}')" + ) + return answer_with_provenance(answer_correct, answer_note, run) R2_TASK: dict[str, Any] = { "id": "R2", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( "In project {project}, how many urgent open work items are there? " "Return exactly one line of the form 'count: N', where N is the integer count." ), - "optimal_calls": 1, - "optimal_tools": {"count_work_items"}, - "alternate_tools": { - "list_work_items", - "search_work_items", - "list_projects", - "list_states", - "get_pql_reference", - }, "needs": {"items"}, "verify": verify_r2, } @@ -96,30 +76,24 @@ async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup """R3: ``item: TITLE`` lines exactly match the seeded due-title set.""" titles = list(ctx.get("r3_due_titles") or []) if not titles: - return False, "no R3 due titles in seed ctx" + return answer_with_provenance(False, "no API-confirmed R3 due titles in seed ctx", run) final_text = get_final_text(run) - if not reports_contract_values(final_text, "item", titles): - return False, f"item contract values={contract_values(final_text, 'item')!r}; want {titles!r}" - return True, f"final text reports exactly {len(titles)} due-this-week assigned items" + answer_correct = reports_contract_values(final_text, "item", titles) + answer_note = ( + f"final text reports exactly {len(titles)} due-this-week assigned items" + if answer_correct + else f"item contract values={contract_values(final_text, 'item')!r}; want {titles!r}" + ) + return answer_with_provenance(answer_correct, answer_note, run) R3_TASK: dict[str, Any] = { "id": "R3", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( "In project {project}, list work items assigned to me that are due this week. " "Return one line per result as 'item: ' and no other 'item:' lines." ), - "optimal_calls": 2, - "optimal_tools": {"get_me", "list_work_items"}, - "alternate_tools": { - "search_work_items", - "count_work_items", - "list_projects", - "get_workspace_members", - "get_pql_reference", - "retrieve_work_item", - }, "needs": {"items"}, "verify": verify_r3, } @@ -130,14 +104,17 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup final_text = get_final_text(run) notes: list[str] = [] ok = True - if not reports_contract_value(final_text, "cycle", CYCLE_CURRENT): + cycle_name = str(ctx.get("r4_cycle_name") or "") + if not cycle_name: ok = False - notes.append(f"cycle values={contract_values(final_text, 'cycle')!r}; want [{CYCLE_CURRENT!r}]") + notes.append("API-confirmed active-cycle name missing from seed ctx") + elif not reports_contract_value(final_text, "cycle", cycle_name): + ok = False + notes.append(f"cycle values={contract_values(final_text, 'cycle')!r}; want [{cycle_name!r}]") else: - notes.append(f"cycle={CYCLE_CURRENT!r}") + notes.append(f"cycle={cycle_name!r}") - active_ids = {str(value) for value in (ctx.get("r4_active_item_ids") or [])} - active_titles = [str(title) for title, item_id in (ctx.get("items") or {}).items() if str(item_id) in active_ids] + active_titles = [str(value) for value in (ctx.get("r4_active_titles") or [])] if not active_titles: ok = False notes.append("no active-cycle titles in seed ctx") @@ -147,35 +124,26 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append(f"{len(active_titles)} active-cycle items") - overdue = str(ctx.get("r4_overdue_title") or "") - expected_overdue = [overdue] if overdue else ["none"] + overdue_titles = [str(value) for value in (ctx.get("r4_overdue_titles") or [])] + expected_overdue = overdue_titles or ["none"] if not reports_contract_values(final_text, "overdue", expected_overdue): ok = False notes.append(f"overdue values={contract_values(final_text, 'overdue')!r}; want {expected_overdue!r}") else: notes.append(f"overdue={expected_overdue!r}") - return ok, "; ".join(notes) + return answer_with_provenance(ok, "; ".join(notes), run) R4_TASK: dict[str, Any] = { "id": "R4", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( "In project {project}, what is in the active cycle, and is anything overdue? " - f"Use these exact contract lines: one 'cycle: {CYCLE_CURRENT}' line, one " + "Use these exact contract lines: one 'cycle: ' line, one " "'item: ' line for every item in that cycle, and one " "'overdue: ' line for every overdue item. If none " "are overdue, use 'overdue: none'." ), - "optimal_calls": 2, - "optimal_tools": {"list_cycles", "list_work_items"}, - "alternate_tools": { - "list_cycle_work_items", - "retrieve_cycle", - "search_work_items", - "list_projects", - "get_pql_reference", - }, "needs": {"items", "cycles"}, "verify": verify_r4, } @@ -183,31 +151,28 @@ async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: """R5: ``comment: TEXT`` lines exactly match the seeded comments.""" - phrases = list(ctx.get("r5_comment_phrases") or R5_COMMENT_PHRASES) + phrases = list(ctx.get("r5_comment_phrases") or []) + if not phrases: + return answer_with_provenance(False, "no API-confirmed R5 comments in seed ctx", run) final_text = get_final_text(run) - if not reports_contract_values(final_text, "comment", phrases): - return False, f"comment values={contract_values(final_text, 'comment')!r}; want {phrases!r}" - return True, f"final text reports exactly {len(phrases)} seeded comments" + answer_correct = reports_contract_values(final_text, "comment", phrases) + answer_note = ( + f"final text reports exactly {len(phrases)} seeded comments" + if answer_correct + else f"comment values={contract_values(final_text, 'comment')!r}; want {phrases!r}" + ) + return answer_with_provenance(answer_correct, answer_note, run) R5_TASK: dict[str, Any] = { "id": "R5", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " "You may summarize in prose, but end with one contract line per comment: " "'comment: '. Copy the comment text exactly and include " "no other 'comment:' lines." ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_work_item_comments"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "retrieve_work_item_by_identifier", - "list_work_item_activities", - "list_projects", - }, "needs": {"items"}, "verify": verify_r5, } @@ -215,78 +180,69 @@ async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: """R6: final text reports the winning project via ``project: NAME``.""" - expected = ctx.get("r6_more_bugs_project") or ctx.get("second_project_name") + expected = str(ctx.get("r6_more_bugs_project") or "") if not expected: - return False, "second project name missing from seed ctx" + return answer_with_provenance(False, "API-confirmed R6 winner missing from seed ctx", run) final_text = get_final_text(run) - if not reports_contract_value(final_text, "project", expected): - return False, f"project values={contract_values(final_text, 'project')!r}; want [{expected!r}]" - return True, f"final text reports project with more bugs {expected!r}" + answer_correct = reports_contract_value(final_text, "project", expected) + answer_note = ( + f"final text reports project with more bugs {expected!r}" + if answer_correct + else f"project values={contract_values(final_text, 'project')!r}; want [{expected!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) R6_TASK: dict[str, Any] = { "id": "R6", - "tags": {"read", "tier1"}, + "tags": {"read"}, "prompt": ( "Across the eval projects created for this run (main project {project} and its " "sibling 'B' project), which project has more open Bug-typed work items? " "Return exactly one line: 'project: '." ), - "optimal_calls": 3, - "optimal_tools": {"list_projects", "list_work_items", "resolve_work_item_type"}, - "alternate_tools": { - "count_work_items", - "search_work_items", - "list_work_item_types", - "retrieve_project", - "get_pql_reference", - }, "needs": {"items", "bug_type", "second_project"}, "verify": verify_r6, } async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R7 (extra): contract names project states or explicitly says unrestricted. - - This preserves the verifier's existing semantic ceiling: the full surface - exposes project states, not authoritative workflow transition evaluation. - The output match itself is structural and exact. - """ + """R7: report the immutable state/group baseline with target-response evidence.""" + baseline = [str(value) for value in (ctx.get("r7_state_pairs") or [])] + if not baseline: + return answer_with_provenance(False, "R7 fixture error: seeded state baseline is empty", run) workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) - names = [(s.name or "").strip() for s in (page.results or []) if (s.name or "").strip()] + states = list(page.results or []) + live = state_name_group_pairs(states) + if Counter(live) != Counter(baseline): + return answer_with_provenance( + False, + f"state oracle was mutated after seeding: live={live!r}; baseline={baseline!r}", + run, + ) + final_text = get_final_text(run) - reported = contract_values(final_text, "transition") - if reported == ["unrestricted"]: - return True, "agent reported unrestricted transitions" - if not reported: - return False, "final text has no 'transition: ' contract lines" - unknown = [value for value in reported if value not in names] - if unknown: - return False, f"transition values {unknown!r} are not exact project state names; have {names!r}" - return True, f"final text reports project state(s) {reported!r} via contract" + if not reports_contract_values(final_text, "state", baseline): + reported = contract_values(final_text, "state") + return answer_with_provenance( + False, + f"state values={reported!r}; want seeded state/group pairs {baseline!r}", + run, + ) + return answer_with_provenance(True, f"final text reports all {len(baseline)} seeded state/group pairs", run) R7_TASK: dict[str, Any] = { "id": "R7", - "tags": {"read", "tier1", "extra"}, + "tags": {"read", "extra"}, "prompt": ( - f"In project {{project}}, what states can the work item '{R1_TITLE}' " - "legally transition to under workflow rules? Return one line per state as " - "'transition: '. If transitions are unrestricted, return " - "exactly 'transition: unrestricted'." + "List every workflow state in project {project} and its group. Return exactly " + "one line per state as 'state: | group: '." ), - # Extra: exercises list_available_transitions. - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "list_states"}, - "alternate_tools": { - "retrieve_work_item", - "search_work_items", - "list_projects", - }, - "needs": {"items"}, + # Extra: exercises project state listing. + "needs": set(), "verify": verify_r7, } diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py index e9800ba2..ee2555d3 100644 --- a/evals/tasks/schema.py +++ b/evals/tasks/schema.py @@ -7,9 +7,10 @@ from plane.errors.errors import HttpError from plane.models.enums import PropertyType -from evals.seed import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE -from evals.tasks.lookups import as_id, find_item_by_name, is_not_found +from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE +from evals.tasks.lookups import as_id, find_item_by_name from evals.tasks.skip import TaskSkipped +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: @@ -36,9 +37,10 @@ async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup or [] ) except HttpError as exc: - if is_not_found(exc): + if is_verifier_not_found(exc): + # A type-scoped 404 is authoritative absence, so it is evidence of a failed end state. return False, "Severity property not found on Bug type (type-scoped list empty/404)" - raise + raise_verifier_read_error("S1", "listing Bug type properties", exc) severity = None for p in props: @@ -71,8 +73,9 @@ async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup ) option_names = {(getattr(o, "name", None) or "").strip() for o in (opts or [])} except HttpError as exc: - if not is_not_found(exc): - raise + if not is_verifier_not_found(exc): + raise_verifier_read_error("S1", "listing Severity options", exc) + # A missing options collection definitively cannot contain the required choices. option_names = set() required = {"critical", "major", "minor"} @@ -85,28 +88,11 @@ async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup S1_TASK: dict[str, Any] = { "id": "S1", - "tags": {"setup", "tier1"}, + "tags": {"setup"}, "prompt": ( "In project {project}, add a Severity dropdown property (options: Critical, " "Major, Minor) to the Bug work item type." ), - "optimal_calls": 3, - "optimal_tools": { - "list_projects", - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "list_work_item_types", - "create_work_item_property_option", - "retrieve_work_item_type", - "list_work_item_properties", - "retrieve_work_item_property", - "manage_work_item_type_properties", - "create_work_item_type", - "import_work_item_types_to_project", - "update_project_features", - }, "needs": {"bug_type"}, "verify": verify_s1, } @@ -123,7 +109,9 @@ async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup try: est = plane.estimates.retrieve(workspace_slug=workspace_slug, project_id=project_id) except Exception as exc: - return False, f"no project estimate: {exc}" + if is_verifier_not_found(exc): + return False, "project estimate not found; requested Fibonacci scale was not created" + raise_verifier_read_error("S2", "retrieving the project estimate", exc) est_id = getattr(est, "id", None) or as_id(est) points = plane.estimates.list_points(workspace_slug=workspace_slug, project_id=project_id, estimate_id=est_id) point_rows = points if isinstance(points, list) else (points.results if hasattr(points, "results") else points) @@ -157,26 +145,11 @@ async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup S2_TASK: dict[str, Any] = { "id": "S2", - "tags": {"setup", "tier1"}, + "tags": {"setup"}, "prompt": ( f"In project {{project}}, add a Fibonacci estimate scale (points 1,2,3,5,8) " f"and set the work item '{W8_TITLE}' to 5 points." ), - "optimal_calls": 5, - "optimal_tools": { - "list_projects", - "create_project_estimate", - "create_project_estimate_points", - "link_estimate_to_project", - "update_work_item", - }, - "alternate_tools": { - "get_project_estimate", - "list_project_estimate_points", - "list_work_items", - "search_work_items", - "update_project_estimate", - }, "needs": {"items"}, "verify": verify_s2, } @@ -200,14 +173,14 @@ async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup features = plane.workspaces.get_features(workspace_slug=workspace_slug) dump = features.model_dump() if hasattr(features, "model_dump") else {} workspace_owns = bool(dump.get("is_work_item_types_enabled")) - except Exception: - workspace_owns = False + except Exception as exc: + raise_verifier_read_error("S3", "reading workspace work-item-type ownership", exc) if workspace_owns: try: wtypes = list(plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []) incident = next((t for t in wtypes if (t.name or "").strip().casefold() == "incident"), None) - except Exception: - pass + except Exception as exc: + raise_verifier_read_error("S3", "listing workspace work-item types", exc) if incident is None: return False, "Incident work item type not found" @@ -221,9 +194,10 @@ async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup or [] ) except HttpError as exc: - if is_not_found(exc): + if is_verifier_not_found(exc): + # A type-scoped 404 is authoritative absence, not an unavailable read. return False, "no properties on Incident type" - raise + raise_verifier_read_error("S3", "listing Incident type properties", exc) required_text = None for p in props: @@ -242,25 +216,11 @@ async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup S3_TASK: dict[str, Any] = { "id": "S3", - "tags": {"setup", "tier1"}, + "tags": {"setup"}, "prompt": ( "In project {project}, create a work item type named 'Incident' and add a " "required text property (e.g. 'Impact summary') on it." ), - "optimal_calls": 3, - "optimal_tools": { - "list_projects", - "resolve_work_item_type", - "create_work_item_property", - }, - "alternate_tools": { - "create_work_item_type", - "list_work_item_types", - "import_work_item_types_to_project", - "list_work_item_properties", - "manage_work_item_type_properties", - "update_project_features", - }, "needs": set(), "verify": verify_s3, } @@ -283,7 +243,7 @@ def _status_of(issue_id: str | None, title: str) -> int | None: row = plane.intake.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=issue_id) return getattr(row, "status", None) except Exception: - # Fall back to list + match title + # Retrieve is optional because the independently authoritative list can resolve the same row. try: rows = plane.intake.list(workspace_slug=workspace_slug, project_id=project_id) results = rows.results if hasattr(rows, "results") else rows @@ -292,8 +252,8 @@ def _status_of(issue_id: str | None, title: str) -> int | None: name = getattr(detail, "name", None) if detail is not None else None if name and name.strip() == title: return getattr(r, "status", None) - except Exception: - return None + except Exception as exc: + raise_verifier_read_error("S4", f"listing intake while resolving {title!r}", exc) return None b_status = _status_of(billing.get("issue_id"), INTAKE_BILLING_TITLE) @@ -314,23 +274,12 @@ def _status_of(issue_id: str | None, title: str) -> int | None: S4_TASK: dict[str, Any] = { "id": "S4", - "tags": {"setup", "tier1"}, + "tags": {"setup"}, "prompt": ( f"In project {{project}}, triage intake: accept the billing request " f"'{INTAKE_BILLING_TITLE}' and reject/decline the spam item " f"'{INTAKE_SPAM_TITLE}'." ), - "optimal_calls": 3, - "optimal_tools": { - "list_intake_work_items", - "update_intake_work_item", - }, - "alternate_tools": { - "retrieve_intake_work_item", - "list_work_items", - "list_projects", - "create_intake_work_item", - }, "needs": {"intake"}, "verify": verify_s4, } @@ -373,8 +322,7 @@ async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append("features.cycles=True") except Exception as exc: - ok = False - notes.append(f"project get_features failed: {exc}") + raise_verifier_read_error("S5", "reading project feature flags", exc) # Workspace customers toggle (is_customer_enabled behind API field ``customers``). try: @@ -397,15 +345,14 @@ async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append("workspace.customers=True") except Exception as exc: - ok = False - notes.append(f"workspace get_features failed: {exc}") + raise_verifier_read_error("S5", "reading workspace customer feature flags", exc) return ok, "; ".join(notes) S5_TASK: dict[str, Any] = { "id": "S5", - "tags": {"setup", "tier1"}, + "tags": {"setup"}, "prompt": ( "Enable cycles and time tracking (worklogs) for project {project}, " "and enable the customers feature for the workspace." @@ -414,14 +361,6 @@ async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup # 1. update_project(cycle_view=True, is_time_tracking_enabled=True) # 2. update_workspace_features(customers=True) # (features PATCH can set cycles→cycle_view but cannot set worklogs.) - "optimal_calls": 2, - "optimal_tools": {"update_project", "update_workspace_features"}, - "alternate_tools": { - "update_project_features", - "list_projects", - "retrieve_project", - "get_features", - }, # Seed leaves project cycles+worklogs and workspace customers off. "needs": {"leave_cycles_worklogs_off"}, "verify": verify_s5, diff --git a/evals/tasks/skip.py b/evals/tasks/skip.py index fc87fde2..29e0a1d8 100644 --- a/evals/tasks/skip.py +++ b/evals/tasks/skip.py @@ -1,12 +1,5 @@ -"""Task skip signal.""" - - -class TaskSkipped(Exception): - """Verifier signals that this task-rep should be recorded as skipped, not failed.""" - - def __init__(self, reason: str) -> None: - super().__init__(reason) - self.reason = reason +"""Backward-compatible task-skip import path.""" +from evals.errors import TaskSkipped as TaskSkipped __all__ = ["TaskSkipped"] diff --git a/evals/tasks/verification.py b/evals/tasks/verification.py new file mode 100644 index 00000000..dd8bfa4f --- /dev/null +++ b/evals/tasks/verification.py @@ -0,0 +1,24 @@ +"""Shared verifier failure semantics.""" + +from __future__ import annotations + +from typing import NoReturn + +from plane.errors.errors import HttpError + + +class VerifierReadError(RuntimeError): + """An infrastructure failure while a verifier was reading authoritative state.""" + + +def is_verifier_not_found(exc: BaseException) -> bool: + """Return whether a verifier read got an authoritative HTTP 404 response.""" + return isinstance(exc, HttpError) and exc.status_code == 404 + + +def raise_verifier_read_error(task_id: str, reading: str, exc: BaseException) -> NoReturn: + """Raise a diagnosable infrastructure error for a required verifier API read.""" + raise VerifierReadError(f"{task_id} verifier read failed while {reading}: {type(exc).__name__}: {exc}") from exc + + +__all__ = ["VerifierReadError", "is_verifier_not_found", "raise_verifier_read_error"] diff --git a/evals/tasks/write.py b/evals/tasks/write.py index 16558778..fc37c98c 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -5,9 +5,9 @@ from typing import Any from plane.errors.errors import HttpError -from plane.models.query_params import RetrieveQueryParams, WorkItemQueryParams +from plane.models.query_params import PaginatedQueryParams, RetrieveQueryParams, WorkItemQueryParams -from evals.seed import ( +from evals.fixtures import ( CYCLE_CURRENT, CYCLE_PAST, MODULE_COMPLETED_TITLES, @@ -19,15 +19,19 @@ W7_URL, W8_TITLE, ) -from evals.tasks.answers import word_boundary +from evals.tasks.answers import normalize_rich_text from evals.tasks.lookups import ( + collect_paginated, find_item_by_name, find_items_by_name, ids, - is_not_found, - state_group, state_name, ) +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error + +W3_COMMENT_TEXT = "Reviewed contrast tokens — needs design pass" +W10_PAGE_NAME = "Eval Runbook" +W10_PAGE_BODY = "Rollback steps for eval harness" async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: @@ -84,62 +88,41 @@ async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup W1_TASK: dict[str, Any] = { "id": "W1", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ( "Create a work item in project {project}: title 'Login page 500s on empty " "password', priority urgent, assign it to me, and add the 'auth' label." ), - "optimal_calls": 4, - "optimal_tools": {"get_me", "list_projects", "list_labels", "create_work_item"}, - "alternate_tools": { - "search_work_items", - "list_states", - "retrieve_project", - "get_workspace_members", - "manage_work_item_assignee", - "manage_work_item_label", - "update_work_item", - "list_work_items", - }, "needs": {"labels"}, "verify": verify_w1, } async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W2: target item is in a completed-group state (prefer name Done).""" + """W2: target item is in the exact state named by the prompt: Done.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] item = find_item_by_name(plane, workspace_slug, project_id, W2_TITLE) if item is None: return False, f"item {W2_TITLE!r} not found" detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) - name = state_name(plane, workspace_slug, project_id, detail.state) - group = state_group(plane, workspace_slug, project_id, detail.state) - if group == "completed" or (name and name.casefold() == "done"): - return True, f"state={name!r} group={group!r}" - return False, f"state={name!r} group={group!r} (want completed/Done)" + name = (state_name(plane, workspace_slug, project_id, detail.state) or "").strip() + if name == "Done": + return True, f"state exactly matches {name!r}" + return False, f"state={name!r} (want exact 'Done')" W2_TASK: dict[str, Any] = { "id": "W2", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": (f"In project {{project}}, move the work item titled '{W2_TITLE}' to the Done state."), - "optimal_calls": 3, - "optimal_tools": {"list_work_items", "list_states", "update_work_item"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "retrieve_state", - "list_projects", - }, "needs": {"items"}, "verify": verify_w2, } async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W3: target item has a comment containing the prompt phrase 'contrast tokens'.""" + """W3: target item has a comment whose normalized text exactly matches the ask.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] item = find_item_by_name(plane, workspace_slug, project_id, W3_TITLE) @@ -153,33 +136,21 @@ async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup results = list(resp.results if hasattr(resp, "results") else resp or []) if not results: return False, "no comments on target item" - phrase = "contrast tokens" - pat = word_boundary(phrase) + expected = normalize_rich_text(W3_COMMENT_TEXT) for c in results: - html = getattr(c, "comment_html", None) or "" - stripped = getattr(c, "comment_stripped", None) or "" - # Some APIs expose plain text under comment_stripped; fall back to html. - blob = f"{stripped}\n{html}" - if pat.search(blob): - return True, f"comment matches {phrase!r}" - return False, f"no comment contains {phrase!r} ({len(results)} comment(s))" + actual = normalize_rich_text(c) + if actual == expected: + return True, f"comment text exactly matches {expected!r}" + actual_texts = [normalize_rich_text(comment) for comment in results] + return False, f"no exact normalized comment {expected!r}; have {actual_texts!r}" W3_TASK: dict[str, Any] = { "id": "W3", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ( - f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' " - "saying 'Reviewed contrast tokens — needs design pass'." + f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' saying '{W3_COMMENT_TEXT}'." ), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "create_work_item_comment"}, - "alternate_tools": { - "search_work_items", - "retrieve_work_item", - "list_work_item_comments", - "list_projects", - }, "needs": {"items"}, "verify": verify_w3, } @@ -197,37 +168,30 @@ async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if triage_id: try: lb = plane.labels.retrieve(workspace_slug=workspace_slug, project_id=project_id, label_id=triage_id) - name = (lb.name or "").strip().casefold() - if name in ("needs-triage", "needs triage"): + name = (lb.name or "").strip() + if name == "needs-triage": return True, f"label id {triage_id} now named {lb.name!r}" - return False, f"label id {triage_id} still named {lb.name!r}" + return False, f"label id {triage_id} named {lb.name!r} (want exact 'needs-triage')" except HttpError as exc: - if not is_not_found(exc): - raise + if not is_verifier_not_found(exc): + raise_verifier_read_error("W4", f"retrieving seeded triage label {triage_id}", exc) + # The seed ID returning 404 proves the requested rename end state does not exist. return False, f"seeded triage label id {triage_id} not found (deleted?)" # Fallback only when seed id is absent from ctx. page = plane.labels.list(workspace_slug=workspace_slug, project_id=project_id) - names = {(lb.name or "").strip().casefold(): (lb.name or "").strip() for lb in (page.results or [])} - if "needs-triage" in names or "needs triage" in names: + names = {(lb.name or "").strip() for lb in (page.results or [])} + if "needs-triage" in names: if "triage" in names: return False, "both triage and needs-triage still present" return True, "label renamed to needs-triage (no seed id; name-scan fallback)" - return False, f"needs-triage not found; labels={sorted(names.values())}" + return False, f"exact needs-triage label not found; labels={sorted(names)}" W4_TASK: dict[str, Any] = { "id": "W4", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ("In project {project}, rename the label 'triage' to 'needs-triage'."), - "optimal_calls": 2, - "optimal_tools": {"list_labels", "update_label"}, - "alternate_tools": { - "retrieve_label", - "create_label", - "delete_label", - "list_projects", - }, "needs": {"labels"}, "verify": verify_w4, } @@ -253,11 +217,11 @@ async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup try: detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) except HttpError as exc: - if is_not_found(exc): - # Deleted OR archived-as-404 — require confirmation via list_archived. + if is_verifier_not_found(exc): + # Retrieve 404 is ambiguous by design; the archived list is an authoritative fallback. need_archive_list.append(str(wid)) continue - raise + raise_verifier_read_error("W5", f"retrieving module work item {wid}", exc) archived_at = getattr(detail, "archived_at", None) if not archived_at: not_archived.append(str(wid)) @@ -265,15 +229,23 @@ async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup arch_ids: set[str] = set() if need_archive_list or not_archived: try: - arch = plane.work_items.list_archived( - workspace_slug=workspace_slug, - project_id=project_id, - params=WorkItemQueryParams(per_page=100), + archived_rows = collect_paginated( + lambda cursor: plane.work_items.list_archived( + workspace_slug=workspace_slug, + project_id=project_id, + params=( + WorkItemQueryParams(cursor=cursor, per_page=100) + if cursor + else WorkItemQueryParams(per_page=100) + ), + ) ) - arch_ids = {str(i.id) for i in (arch.results or [])} + arch_ids = {str(i.id) for i in archived_rows} except Exception as exc: if need_archive_list: - return False, f"list_archived failed while confirming 404 items: {exc}" + raise_verifier_read_error("W5", "listing archived items to resolve retrieve 404s", exc) + # Optional cross-check only: successful retrieves already prove these rows are unarchived. + pass # 404s only count as archived if present on the archived list (deletes fail). for wid in need_archive_list: @@ -288,20 +260,8 @@ async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup W5_TASK: dict[str, Any] = { "id": "W5", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": (f"In project {{project}}, archive all completed work items in the module '{MODULE_NAME}'."), - "optimal_calls": 5, # list_modules + list_module_work_items + 3× archive - "optimal_tools": { - "list_modules", - "list_module_work_items", - "manage_work_item_archive", - }, - "alternate_tools": { - "list_work_items", - "retrieve_module", - "list_projects", - "list_states", - }, "needs": {"module"}, "verify": verify_w5, } @@ -321,7 +281,12 @@ async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup past_id = ctx.get("cycle_past_id") or (ctx.get("cycles") or {}).get(CYCLE_PAST) cur_id = ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) if not past_id: - return False, "Sprint 12 id missing from seed" + raise RuntimeError(f"W6 fixture error: {CYCLE_PAST} id missing from seed") + if not cur_id: + raise RuntimeError(f"W6 fixture error: {CYCLE_CURRENT} id missing from seed") + unfinished = list(ctx.get("w6_unfinished_titles") or []) + if not unfinished: + raise RuntimeError(f"W6 fixture error: expected unfinished items for {CYCLE_CURRENT} are empty") notes: list[str] = [] ok = True past = plane.cycles.retrieve(workspace_slug=workspace_slug, project_id=project_id, cycle_id=past_id) @@ -356,47 +321,34 @@ async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup f"(want end_date={today!r} or archived_at or progress_snapshot)" ) - unfinished = list(ctx.get("w6_unfinished_titles") or []) - if cur_id and unfinished: - try: - on13 = plane.cycles.list_work_items( - workspace_slug=workspace_slug, - project_id=project_id, - cycle_id=cur_id, - params=WorkItemQueryParams(per_page=100), - ) - names = {(i.name or "").strip() for i in (on13.results or [])} - missing = [t for t in unfinished if t not in names] - if missing: - ok = False - notes.append(f"unfinished not on Sprint 13: {missing}") - else: - notes.append(f"{len(unfinished)} unfinished on Sprint 13") - except Exception as exc: - notes.append(f"list Sprint 13 items failed: {exc}") + try: + on13 = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=cur_id, + params=WorkItemQueryParams(per_page=100), + ) + names = {(i.name or "").strip() for i in (on13.results or [])} + except Exception as exc: + if is_verifier_not_found(exc): + return False, f"{CYCLE_CURRENT} not found while checking unfinished-item rollover" + raise_verifier_read_error("W6", f"listing {CYCLE_CURRENT} work items", exc) + missing = [t for t in unfinished if t not in names] + if missing: + ok = False + notes.append(f"unfinished not on Sprint 13: {missing}") + else: + notes.append(f"{len(unfinished)} unfinished on Sprint 13") return ok, "; ".join(notes) W6_TASK: dict[str, Any] = { "id": "W6", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ( f"In project {{project}}, '{CYCLE_PAST}' is wrapping up. Close it and make sure " f"its unfinished work items end up on '{CYCLE_CURRENT}'." ), - "optimal_calls": 4, - "optimal_tools": { - "list_cycles", - "transfer_cycle_work_items", - "complete_cycle", - }, - "alternate_tools": { - "list_cycle_work_items", - "manage_cycle_work_items", - "update_cycle", - "list_work_items", - "list_projects", - }, # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — # Plane rejects every edit to an ended cycle. See _seed_cycles. "needs": {"items", "cycles", "cycles_open_past"}, @@ -444,8 +396,7 @@ async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append("blocking relation present") except Exception as exc: - ok = False - notes.append(f"dependencies list failed: {exc}") + raise_verifier_read_error("W7", f"listing dependencies for source item {src.id}", exc) # Links try: @@ -462,43 +413,25 @@ async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup else: notes.append("reference URL present") except Exception as exc: - ok = False - notes.append(f"links list failed: {exc}") + raise_verifier_read_error("W7", f"listing links for source item {src.id}", exc) return ok, "; ".join(notes) W7_TASK: dict[str, Any] = { "id": "W7", - "tags": {"write", "tier1"}, + "tags": {"write"}, "prompt": ( f"In project {{project}}, mark the work item '{W7_SOURCE_TITLE}' as blocking " f"'{W7_TARGET_TITLE}', and add the reference URL {W7_URL} on the blocking item." ), - "optimal_calls": 3, - "optimal_tools": { - "list_work_items", - "create_work_item_relation", - "create_work_item_link", - }, - "alternate_tools": { - "search_work_items", - "list_work_item_relations", - "list_work_item_relation_definitions", - "list_work_item_links", - "retrieve_work_item", - }, "needs": {"items"}, "verify": verify_w7, } async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W8: work log of exactly 120 minutes exists on the target item. - - Note: plane-sdk Create work log has no logged-date field — 'yesterday' in the - prompt cannot be asserted; only duration is verified. - """ + """W8: work log of exactly 120 minutes exists on the target item.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] item = find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) @@ -518,16 +451,8 @@ async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup W8_TASK: dict[str, Any] = { "id": "W8", - "tags": {"write", "tier1"}, - "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}' for yesterday."), - "optimal_calls": 2, - "optimal_tools": {"list_work_items", "create_work_log"}, - "alternate_tools": { - "search_work_items", - "list_work_logs", - "retrieve_work_item", - "list_projects", - }, + "tags": {"write"}, + "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}'."), "needs": {"items"}, "verify": verify_w8, } @@ -541,6 +466,7 @@ def _time_tracking_enabled(plane: Any, workspace_slug: str, project_id: str) -> try: project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) except Exception: + # Optional diagnosis only: the already-successful worklog read determines failure. return None value = getattr(project, "is_time_tracking_enabled", None) return None if value is None else bool(value) @@ -569,8 +495,9 @@ async def verify_w11(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu ) except Exception as exc: # The list endpoint is gated by the same toggle, so this is the diagnosis itself. - if not is_not_found(exc): - raise + if not is_verifier_not_found(exc): + raise_verifier_read_error("W11", f"listing work logs for item {item.id}", exc) + # This endpoint's documented gate is a 404, which proves time tracking remains disabled. return False, f"work log list still refused, so time tracking was never enabled ({exc})" rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] @@ -590,24 +517,13 @@ async def verify_w11(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu W11_TASK: dict[str, Any] = { "id": "W11", - "tags": {"write", "gate", "tier1"}, + "tags": {"write", "gate"}, "prompt": ( f"In project {{project}}, log 2 hours of work on the item titled '{W11_TITLE}'. " "Time tracking may be switched off for this project; you have permission to turn " "it on if you need to." ), # enable time tracking → find the item → create the work log. - "optimal_calls": 3, - "optimal_tools": {"update_project", "list_work_items", "create_work_log"}, - "alternate_tools": { - "update_project_features", - "get_features", - "retrieve_project", - "search_work_items", - "list_work_logs", - "retrieve_work_item", - "list_projects", - }, "needs": {"items", "leave_worklogs_off"}, "verify": verify_w11, } @@ -641,59 +557,70 @@ async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup W9_TASK: dict[str, Any] = { "id": "W9", - "tags": {"write", "tier1", "extra"}, + "tags": {"write", "extra"}, "prompt": ( - "In project {project}, set priority to high on these three work items in one " - "batch: 'Checkout times out on 3DS challenge', " + "In project {project}, set priority to high on these three work items: " + "'Checkout times out on 3DS challenge', " "'Session cookie not rotated after login', " "'Inventory count goes negative under load'." ), - # Extra: exercises bulk_update_work_items (not in original DESIGN 20). - "optimal_calls": 4, - "optimal_tools": { - "list_work_items", - "update_work_item", - }, - "alternate_tools": { - "search_work_items", - "list_projects", - "retrieve_work_item", - }, + # Extra: call distributions describe whether agents batch this multi-item mutation. "needs": {"items"}, "verify": verify_w9, } async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """W10 (extra): project page named Eval Runbook exists.""" + """W10 (extra): named project page has the exact normalized requested body.""" workspace_slug = ctx["workspace_slug"] project_id = ctx["project_id"] try: - resp = plane.pages.list_project_pages(workspace_slug=workspace_slug, project_id=project_id) - rows = resp.results if hasattr(resp, "results") else resp + rows = collect_paginated( + lambda cursor: plane.pages.list_project_pages( + workspace_slug=workspace_slug, + project_id=project_id, + params=( + PaginatedQueryParams(cursor=cursor, per_page=100) if cursor else PaginatedQueryParams(per_page=100) + ), + ) + ) except Exception as exc: - return False, f"list pages failed: {exc}" - names = {(getattr(p, "name", None) or "").strip() for p in (rows or [])} - if "Eval Runbook" not in names: - return False, f"page 'Eval Runbook' missing; have {sorted(names)}" - return True, "page Eval Runbook present" + raise_verifier_read_error("W10", "listing project pages", exc) + candidates = [page for page in rows if (getattr(page, "name", None) or "").strip() == W10_PAGE_NAME] + if not candidates: + names = sorted({(getattr(page, "name", None) or "").strip() for page in rows}) + return False, f"page {W10_PAGE_NAME!r} missing; have {names}" + actual_bodies: list[str] = [] + for page in candidates: + page_id = getattr(page, "id", None) + if not page_id: + actual_bodies.append("") + continue + try: + detail = plane.pages.retrieve_project_page( + workspace_slug=workspace_slug, + project_id=project_id, + page_id=page_id, + ) + except Exception as exc: + if is_verifier_not_found(exc): + actual_bodies.append(f"") + continue + raise_verifier_read_error("W10", f"retrieving project page {page_id}", exc) + actual = normalize_rich_text(detail) + actual_bodies.append(actual) + if actual == W10_PAGE_BODY: + return True, f"page {W10_PAGE_NAME!r} has exact normalized body" + return False, f"page {W10_PAGE_NAME!r} body mismatch: have {actual_bodies!r}; want {W10_PAGE_BODY!r}" W10_TASK: dict[str, Any] = { "id": "W10", - "tags": {"write", "tier1", "extra"}, + "tags": {"write", "extra"}, "prompt": ( - "In project {project}, create a project page named 'Eval Runbook' with body " - "text 'Rollback steps for eval harness'." + f"In project {{project}}, create a project page named '{W10_PAGE_NAME}' with body text '{W10_PAGE_BODY}'." ), # Extra: exercises pages family (create_page / get_page). - "optimal_calls": 2, - "optimal_tools": {"list_projects", "create_page"}, - "alternate_tools": { - "list_pages", - "retrieve_page", - "attach_page_to_work_item", - }, "needs": set(), "verify": verify_w10, } @@ -715,6 +642,9 @@ async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tu __all__ = [ + "W3_COMMENT_TEXT", + "W10_PAGE_BODY", + "W10_PAGE_NAME", "W11_TITLE", "WRITE_TASKS", "verify_w1", diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 09603b4a..9c97dede 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -35,7 +35,7 @@ "C2", } -EXTRA_IDS = {"W9", "W10", "R7", "S5", "W11"} # bulk, pages, transitions, features, gate recovery +EXTRA_IDS = {"W9", "W10", "R7", "S5", "W11"} # bulk, pages, state inventory, features, gate recovery ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} @@ -81,61 +81,58 @@ "L5", ) -# "eea5abf36382" before CATALOG_REVISION entered the payload; "232036625e00" at revision 1. +# "eea5abf36382" before CATALOG_REVISION entered the payload; "232036625e00" at revision 1; +# "9c148461e674" at revision 3 before fixture names joined the payload. +# "013109dc1c4c" at revision 3 after fixture names joined the payload. +# "9f2c2feb2e24" at revision 4 before read provenance and randomised truth. +# "7b8dc6bd2f8f" at revision 5 before unverifiable W8/W9 asks were removed. +# "77230f96962d" at revision 6 before target-entity response evidence. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "3e9194740d73" +PINNED_SYNTHETIC_BATTERY = "e059523c9f3d" -def test_catalog_behaviours(): - def test_catalog_includes_design_and_extras(): - ids = {t["id"] for t in TASKS} - assert DESIGN_IDS.issubset(ids), f"missing DESIGN ids: {DESIGN_IDS - ids}" - assert EXTRA_IDS.issubset(ids), f"missing extra ids: {EXTRA_IDS - ids}" - assert ID_IN_HAND_IDS.issubset(ids), f"missing I-class: {ID_IN_HAND_IDS - ids}" - assert LONG_TAIL_IDS.issubset(ids), f"missing L-class: {LONG_TAIL_IDS - ids}" - assert len(TASKS) >= 20 - - def test_catalog_id_order_is_pinned(): +@pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) +def test_catalog_behaviours(case): + if case == "id-order": assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER - - test_catalog_includes_design_and_extras() - test_catalog_id_order_is_pinned() - - -def test_get_tasks_behaviours(): - def test_get_tasks_all_and_filter(): - all_t = get_tasks(None) - assert len(all_t) == len(TASKS) - subset = get_tasks(["R1", "W9", "C2"]) - assert [t["id"] for t in subset] == ["R1", "W9", "C2"] - - def test_get_tasks_unknown_exits(): + return + + ids = {task["id"] for task in TASKS} + for expected, label in ( + (DESIGN_IDS, "DESIGN"), + (EXTRA_IDS, "extra"), + (ID_IN_HAND_IDS, "I-class"), + (LONG_TAIL_IDS, "L-class"), + ): + assert expected.issubset(ids), f"missing {label}: {expected - ids}" + assert len(TASKS) >= 20 + + +@pytest.mark.parametrize("case", ["all-and-filter", "unknown-id"]) +def test_get_tasks_behaviours(case): + if case == "unknown-id": with pytest.raises(SystemExit): get_tasks(["NOPE"]) + return - test_get_tasks_all_and_filter() - test_get_tasks_unknown_exits() + assert len(get_tasks(None)) == len(TASKS) + assert [task["id"] for task in get_tasks(["R1", "W9", "C2"])] == ["R1", "W9", "C2"] -def test_task_behaviours(): - def test_task_schema_invariants(): - for t in TASKS: - assert t["id"] - assert isinstance(t["tags"], set) - assert "{project}" in t["prompt"] or t["id"] in NO_PROJECT_PROMPT_IDS - assert isinstance(t["optimal_tools"], set) and t["optimal_tools"] - assert isinstance(t["alternate_tools"], set) - assert t["optimal_tools"].isdisjoint(t["alternate_tools"]), t["id"] - assert callable(t["verify"]) - assert isinstance(t.get("needs"), set) - - def test_task_author_default(): +@pytest.mark.parametrize("case", ["schema-invariants", "author-default"]) +def test_task_behaviours(case): + if case == "author-default": assert task_author({}) == "claude" assert task_author({"author": "alice"}) == "alice" + return - test_task_schema_invariants() - test_task_author_default() + for task in TASKS: + assert task["id"] + assert isinstance(task["tags"], set) + assert "{project}" in task["prompt"] or task["id"] in NO_PROJECT_PROMPT_IDS + assert callable(task["verify"]) + assert isinstance(task.get("needs"), set) def test_debias_tasks_author(): @@ -156,6 +153,18 @@ def test_w6_seeds_an_open_cycle(): assert "cycles" in TASKS_BY_ID["W6"]["needs"] +def test_prompts_do_not_ask_for_unverifiable_w8_date_or_w9_batching(): + from plane.models.work_items import CreateWorkItemWorkLog, WorkItemWorkLog + + w8_prompt = str(TASKS_BY_ID["W8"]["prompt"]) + w9_prompt = str(TASKS_BY_ID["W9"]["prompt"]) + authoritative_date_fields = {"logged_at", "logged_date", "work_date", "date"} + assert not authoritative_date_fields.intersection(CreateWorkItemWorkLog.model_fields) + assert not authoritative_date_fields.intersection(WorkItemWorkLog.model_fields) + assert "yesterday" not in w8_prompt.casefold() + assert "in one batch" not in w9_prompt.casefold() + + def test_verifiers_are_async_and_importable(): modules = { "R": "read", @@ -179,114 +188,69 @@ def test_tasks_module_has_no_hardcoded_uuids(): assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) -def test_prompt_bind_behaviours(): - def test_prompt_bind_strict_empty_raises(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import PromptBindError, format_task_prompt +@pytest.mark.parametrize( + "case", + ["strict-empty", "strict-exception", "dry-run-markers", "strict-success"], +) +def test_prompt_bind_behaviours(case): + from evals.tasks.prompts import PromptBindError, format_task_prompt - t = TASKS_BY_ID["I1"] + task = TASKS_BY_ID["I1"] + if case == "strict-empty": with pytest.raises(PromptBindError): - format_task_prompt(t, {"project_name": "P", "items": {}}, strict=True) - - def test_prompt_bind_strict_exception_raises(): - from evals.tasks.prompts import PromptBindError, format_task_prompt + format_task_prompt(task, {"project_name": "P", "items": {}}, strict=True) + elif case == "strict-exception": def boom(_ctx): raise RuntimeError("seed broken") - task = { - "id": "X", - "prompt": "do {work_item_id}", - "prompt_bind": boom, - } + custom = {"id": "X", "prompt": "do {work_item_id}", "prompt_bind": boom} with pytest.raises(PromptBindError, match="prompt_bind failed"): - format_task_prompt(task, {"project_name": "P"}, strict=True) - - def test_prompt_bind_dry_run_markers(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] - text = format_task_prompt(t, {"project_name": "EVAL x"}, strict=False) - assert "" in text - assert "EVAL x" in text - - def test_prompt_bind_strict_success(): - from evals.tasks.catalog import TASKS_BY_ID - from evals.tasks.prompts import format_task_prompt - - t = TASKS_BY_ID["I1"] + format_task_prompt(custom, {"project_name": "P"}, strict=True) + elif case == "dry-run-markers": + text = format_task_prompt(task, {"project_name": "EVAL x"}, strict=False) + assert "" in text and "EVAL x" in text + else: text = format_task_prompt( - t, + task, {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, strict=True, ) - assert "uuid-abc" in text - assert "<" not in text - - test_prompt_bind_strict_empty_raises() - test_prompt_bind_strict_exception_raises() - test_prompt_bind_dry_run_markers() - test_prompt_bind_strict_success() - - -def test_battery_fingerprint_behaviours(): - def test_battery_fingerprint_stable_and_sensitive(): - t1 = { - "id": "A", - "prompt": "p1 {project}", - "optimal_tools": {"b", "a"}, - "alternate_tools": {"c"}, - "optimal_calls": 2, - } - t2 = { - "id": "B", - "prompt": "p2", - "optimal_tools": {"x"}, - "alternate_tools": set(), - "optimal_calls": 1, - } - # Order of list must not matter (sorted by id). - h1 = battery_fingerprint([t2, t1]) - h2 = battery_fingerprint([t1, t2]) - assert h1 == h2 == PINNED_SYNTHETIC_BATTERY - assert len(h1) == 12 - - t1_edit = {**t1, "prompt": "p1 edited {project}"} - assert battery_fingerprint([t1_edit, t2]) != PINNED_SYNTHETIC_BATTERY - - # Subset of selected tasks → different fingerprint (documented ceiling). - assert battery_fingerprint([t1]) != PINNED_SYNTHETIC_BATTERY - - def test_battery_fingerprint_catalog_is_nonempty(): - from evals.tasks.catalog import TASKS - - fp = battery_fingerprint() - assert len(fp) == 12 - assert battery_fingerprint(list(TASKS)) == fp - - def test_battery_fingerprint_changes_with_new_debias_tasks(): - from evals.tasks.catalog import TASKS, TASKS_BY_ID + assert "uuid-abc" in text and "<" not in text + +@pytest.mark.parametrize( + "case", + ["stable-and-sensitive", "catalog-nonempty", "debias-tasks-change-hash"], +) +def test_battery_fingerprint_behaviours(case): + if case == "catalog-nonempty": + fingerprint = battery_fingerprint() + assert len(fingerprint) == 12 + assert battery_fingerprint(list(TASKS)) == fingerprint + return + if case == "debias-tasks-change-hash": full = battery_fingerprint() - without_debias = [t for t in TASKS if not str(t.get("id", "")).startswith(("I", "L"))] + without_debias = [task for task in TASKS if not str(task.get("id", "")).startswith(("I", "L"))] assert without_debias, "pre-debias catalog should be non-empty" reduced = battery_fingerprint(without_debias) assert reduced != full - # Single new task also moves the hash relative to a reduced set. assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced + return - test_battery_fingerprint_stable_and_sensitive() - test_battery_fingerprint_catalog_is_nonempty() - test_battery_fingerprint_changes_with_new_debias_tasks() + task_a = {"id": "A", "prompt": "p1 {project}"} + task_b = {"id": "B", "prompt": "p2"} + assert battery_fingerprint([task_b, task_a]) == battery_fingerprint([task_a, task_b]) == PINNED_SYNTHETIC_BATTERY + assert battery_fingerprint([{**task_a, "prompt": "p1 edited {project}"}, task_b]) != PINNED_SYNTHETIC_BATTERY + assert battery_fingerprint([task_a]) != PINNED_SYNTHETIC_BATTERY def test_revision_bump_changes_the_fingerprint_for_an_unchanged_catalog(): """A fixture/verifier correction is expressible even though the hash ignores them. - The per-task payload deliberately omits ``needs`` and verifier bodies, so without - the revision a corrected seeder would keep the old fingerprint and go on asserting - that results answering a different question are comparable. + The per-task payload deliberately omits verifier bodies, so without the revision a + corrected verifier could keep the old fingerprint and go on asserting that results + graded against a different contract are comparable. """ from evals.tasks import catalog @@ -307,12 +271,20 @@ def test_fingerprint_records_the_revision_transition(): """Pin the current value, so a future change is read as intentional, not drift. ``d546d3181bdb`` was the fingerprint before the revision field existed (batteries 6-8). - Revision 2 is the feature-exclusion correction, which redefines what S5 asks without - touching any prompt or tool set — exactly the change the hash could not otherwise see. - Asserting the constant rather than merely 'it changed' is what makes an unexplained - future move visible. + Revision 3 drops author-declared tool sets and call floors and adds fixture names, so + the hash covers what the agent was asked and what it was given — never how anyone + expected it to answer. Its final full-catalog value was ``d89173c744cc``. Revision 4 + rewrites R7 to replace its unconditional-pass transition question with an exact + state-and-group listing; its full-catalog value was ``0c9b6fc0405e``. Revision 5 + adds successful Plane-call provenance and randomised API-confirmed seed truth to the + read family; its full-catalog value was ``075bbd409f15``. Revision 6 removes W8's + unverifiable logged-date ask and W9's unverifiable batching ask, and tightens the + affected end-state verifiers; its full-catalog value was ``ccf39203f656``. Revision 7 + binds read provenance to target-entity response evidence and gives C2/R7 randomised, + immutable seed-time oracles. Results across these transitions are not comparable. + Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 2 - assert battery_fingerprint() == "4fb3a34a7231" + assert CATALOG_REVISION == 7 + assert battery_fingerprint() == "9ea76bf22ba0" diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py index 5058ead8..d64e3c5e 100644 --- a/tests/evals/tasks/test_debias_verifiers.py +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -6,6 +6,7 @@ from types import SimpleNamespace from typing import Any +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import W2_TITLE from evals.tasks.debias import ( I1_TITLE, @@ -38,7 +39,18 @@ def __init__(self, results: list[Any] | None = None): def _run(text: str = "") -> dict[str, Any]: - return {"final_text": text, "calls": []} + return { + "final_text": text, + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + "call_source": "test", + "evidence_trace_available": True, + } def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: @@ -166,7 +178,12 @@ async def _go(): def plane_for(states): return _WIRetrievePlane(by_id={"wi-2": SimpleNamespace(id="wi-2", state=BACKLOG)}, states=states) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {W2_TITLE: "wi-2"}} + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {W2_TITLE: "wi-2"}, + "i2_state_name": "Backlog", + } cases = [ ("untouched: empty answer", [BACKLOG], "", False), ("names a different state", [BACKLOG, DONE], "Done", False), @@ -242,7 +259,12 @@ def test_l1_grades_the_duration_contract_not_the_prose(): """ async def _go(): - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}} + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L1_TITLE: "wi-l1"}, + "l1_expected_summary_ids": ["wi-l1"], + } cases = [ ("untouched: no worklog", [], None, "", False, ()), ("wrong duration 120", [120], ["wi-l1"], "Logged 120 minutes; summary ok.", False, ("90",)), @@ -280,12 +302,25 @@ async def _go(): ) assert "duration" in note.lower() or "90" in note or "1.5" in note, note + mutated_ok, mutated_note = await verify_l1( + _L1Plane([90], summary_ids=["wi-l1", "agent-added"]), + dict(ctx), + _run("logged-minutes: 90\nsummary-work-item-id: wi-l1\nsummary-work-item-id: agent-added"), + ) + assert mutated_ok is False + assert "mutated beyond the seeded oracle" in mutated_note + return asyncio.run(_go()) def test_l2_counts_activities_through_the_contract_only(): async def _go(): - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L2_TITLE: "wi-l2"}} + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L2_TITLE: "wi-l2"}, + "l2_activity_count": 3, + } cases = [ ("untouched: empty answer", "", False), ("contract matches truth 3", "Saw some history.\ncount: 3", True), @@ -343,18 +378,23 @@ async def _go(): return asyncio.run(_go()) -def test_l5_accepts_a_zero_count_only_through_the_contract(): +def test_l5_accepts_the_api_confirmed_count_only_through_the_contract(): async def _go(): - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {L5_TITLE: "wi-l5"}} + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L5_TITLE: "wi-l5"}, + "l5_attachment_count": 2, + } cases = [ ("untouched: empty answer", "", False), - ("bare zero as the whole answer", "0", True), - ("multiline ending in the contract", "No files on this work item.\ncount: 0", True), - ("prose without contract", "There are 0 attachments.", False), + ("bare count as the whole answer", "2", True), + ("multiline ending in the contract", "Two files on this work item.\ncount: 2", True), + ("prose without contract", "There are 2 attachments.", False), ("contract with the wrong count", "count: 10", False), ] for label, text, want in cases: - ok, note = await verify_l5(_L5Plane(0), dict(ctx), _run(text)) + ok, note = await verify_l5(_L5Plane(2), dict(ctx), _run(text)) assert ok is want, f"{label}: {note}" return asyncio.run(_go()) diff --git a/tests/evals/tasks/test_gate_recovery.py b/tests/evals/tasks/test_gate_recovery.py index 028a2a59..49aaf76f 100644 --- a/tests/evals/tasks/test_gate_recovery.py +++ b/tests/evals/tasks/test_gate_recovery.py @@ -17,6 +17,7 @@ from plane.errors.errors import HttpError from evals.tasks.catalog import TASKS_BY_ID +from evals.tasks.verification import VerifierReadError from evals.tasks.write import W11_TITLE, verify_w11 WORKLOG_DISABLED = HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}) @@ -101,7 +102,7 @@ def test_a_do_nothing_agent_fails(): def test_an_unexpected_error_is_not_swallowed_as_a_disabled_feature(): """Only the 'worklog disabled' 404 is read as the obstacle; anything else is a bug.""" plane = _plane(logs=HttpError("Server error", 500, {"error": "boom"})) - with pytest.raises(HttpError): + with pytest.raises(VerifierReadError, match="W11 verifier read failed while listing work logs"): asyncio.run(verify_w11(plane, dict(CTX), {"final_text": ""})) @@ -112,4 +113,3 @@ def test_task_seeds_time_tracking_off_and_authorises_turning_it_on(): # Without explicit permission, an agent that declines to change project-wide config is # arguably behaving better, and scoring the enable as success would reward overreach. assert "permission" in task["prompt"].lower() - assert task["optimal_calls"] > TASKS_BY_ID["W8"]["optimal_calls"], "recovery costs a call" diff --git a/tests/evals/tasks/test_lookups.py b/tests/evals/tasks/test_lookups.py new file mode 100644 index 00000000..f63a6552 --- /dev/null +++ b/tests/evals/tasks/test_lookups.py @@ -0,0 +1,23 @@ +"""Shape-tolerance tests for shared verifier lookups.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from evals.tasks.lookups import state_group + + +@pytest.mark.parametrize( + "response", + [ + [SimpleNamespace(id="state-1", group="started")], + SimpleNamespace(results=[SimpleNamespace(id="state-1", group="started")]), + ], + ids=["raw-list", "paginated-page"], +) +def test_state_group_accepts_raw_and_paginated_list_shapes(response): + plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: response)) + + assert state_group(plane, "ws", "project", "state-1") == "started" diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py index 538d7eca..a0d7c4b7 100644 --- a/tests/evals/tasks/test_output_contracts.py +++ b/tests/evals/tasks/test_output_contracts.py @@ -6,10 +6,15 @@ from types import SimpleNamespace from typing import Any +import pytest + +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, W2_TITLE, W8_TITLE from evals.tasks.cross import verify_c2 -from evals.tasks.read import verify_r1, verify_r2, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.debias import verify_i2, verify_l2, verify_l5 +from evals.tasks.read import verify_r1, verify_r2, verify_r3, verify_r4, verify_r5, verify_r6, verify_r7 from evals.tasks.write import verify_w2, verify_w4, verify_w8 +from tests.evals.conftest import case_params class _Page: @@ -19,8 +24,23 @@ def __init__(self, results: list[Any] | None = None): self.next_cursor = None -def _run(text: str = "") -> dict[str, Any]: - return {"final_text": text, "calls": []} +def _run(text: str = "", *, calls: list[dict[str, Any]] | None = None) -> dict[str, Any]: + return { + "final_text": text, + "calls": ( + [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ] + if calls is None + else calls + ), + "call_source": "test", + "evidence_trace_available": True, + } def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: @@ -71,53 +91,75 @@ def __init__(self, durations: list[int]): ) -def test_r2_behaviours(): - def test_r2_written_number_prose_fails_and_count_contract_passes(): - async def _go(): - state = SimpleNamespace(id="started", name="Started", group="started") - items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] - plane = SimpleNamespace( - states=SimpleNamespace(list=lambda **kwargs: _Page([state])), - work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), - ) - ctx = {"workspace_slug": "ws", "project_id": "project"} +class _C2Plane: + def __init__(self, changelog_html: str = "", error: Exception | None = None, release_name: str = "1.2.0"): + def retrieve(**kwargs): + if error is not None: + raise error + return SimpleNamespace(description_html=changelog_html) - prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) - contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) + self.releases = SimpleNamespace( + retrieve=lambda **kwargs: SimpleNamespace(name=release_name), + changelog=SimpleNamespace(retrieve=retrieve), + ) - assert prose_ok is False - assert contract_ok is True, note - return asyncio.run(_go()) +def _r2_written_number_prose_fails_and_count_contract_passes(): + async def _go(): + state = SimpleNamespace(id="started", name="Started", group="started") + items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] + plane = SimpleNamespace( + states=SimpleNamespace(list=lambda **kwargs: _Page([state])), + work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), + ) + ctx = {"workspace_slug": "ws", "project_id": "project", "r2_urgent_open_count": 4} - def test_r2_rejects_a_count_that_disagrees_with_the_api(): - async def _go(): - from evals.tasks.read import verify_r2 as _vr2 + prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) + contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) - urgent = [_item(str(i), "x", priority="urgent", state=SimpleNamespace(group="started")) for i in range(4)] + assert prose_ok is False + assert contract_ok is True, note - class Plane: - work_items = SimpleNamespace(list=lambda **kw: _Page(urgent)) - states = SimpleNamespace( - list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) - ) + return asyncio.run(_go()) - ok, note = await _vr2(Plane(), {"workspace_slug": "ws", "project_id": "p1"}, _run("0")) - assert ok is False, note - return asyncio.run(_go()) +def _r2_rejects_a_count_that_disagrees_with_the_api(): + async def _go(): + from evals.tasks.read import verify_r2 as _vr2 - test_r2_written_number_prose_fails_and_count_contract_passes() - test_r2_rejects_a_count_that_disagrees_with_the_api() + urgent = [_item(str(i), "x", priority="urgent", state=SimpleNamespace(group="started")) for i in range(4)] + + class Plane: + work_items = SimpleNamespace(list=lambda **kw: _Page(urgent)) + states = SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) + ) + + ctx = {"workspace_slug": "ws", "project_id": "p1", "r2_urgent_open_count": 4} + ok, note = await _vr2(Plane(), ctx, _run("0")) + assert ok is False, note + + return asyncio.run(_go()) + + +@pytest.mark.parametrize( + "case", + case_params( + _r2_written_number_prose_fails_and_count_contract_passes, + _r2_rejects_a_count_that_disagrees_with_the_api, + ), +) +def test_r2_behaviours(case): + case() def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): async def _go(): overdue = "Session cookie not rotated after login" ctx = { - "items": {R1_TITLE: "item-1", overdue: "item-2"}, - "r4_active_item_ids": ["item-1", "item-2"], - "r4_overdue_title": overdue, + "r4_cycle_name": CYCLE_CURRENT, + "r4_active_titles": [R1_TITLE, overdue], + "r4_overdue_titles": [overdue], } text = f"cycle: {CYCLE_CURRENT}\nitem: {R1_TITLE}\nitem: {overdue}\noverdue: {overdue}" @@ -159,21 +201,193 @@ async def _go(): return asyncio.run(_go()) -def test_r7_transition_contract_is_structural(): +def test_read_provenance_matrix_and_canary_coverage(): + async def _go(): + ctx = {"r2_urgent_open_count": 4} + no_call_ok, no_call_note = await verify_r2(object(), ctx, _run("count: 4", calls=[])) + assert no_call_ok is False + assert "answer_correct=true" in no_call_note + assert "provenance=missing" in no_call_note + + successful_ok, successful_note = await verify_r2(object(), ctx, _run("count: 4")) + assert successful_ok is True, successful_note + assert "answer_correct=true" in successful_note + assert "provenance=observed" in successful_note + + unrelated_ok, unrelated_note = await verify_r2( + object(), + ctx, + _run( + "count: 4", + calls=[{"tool": "plane_call", "is_error": False, "observed_sentinels": []}], + ), + ) + assert unrelated_ok is False + assert "answer_correct=true" in unrelated_note + assert "0 evidence-bearing of 1 successful" in unrelated_note + + failed_call_ok, failed_call_note = await verify_r2( + object(), + ctx, + _run("count: 4", calls=[{"tool": "plane_call", "is_error": True}]), + ) + assert failed_call_ok is False + assert "answer_correct=true" in failed_call_note + assert "0 evidence-bearing of 0 successful" in failed_call_note + + wrong_ok, wrong_note = await verify_r2(object(), ctx, _run("count: 3")) + assert wrong_ok is False + assert "answer_correct=false" in wrong_note + assert "provenance=observed" in wrong_note + + unavailable_ok, unavailable_note = await verify_r2( + object(), + ctx, + {"final_text": "count: 4"}, + ) + assert unavailable_ok is False + assert "provenance=unavailable" in unavailable_note + + incomplete_ok, incomplete_note = await verify_r2( + object(), + ctx, + { + **_run("count: 4"), + "driver_notes": ["proxy_sidecar_incomplete:skipped_rows=1"], + }, + ) + assert incomplete_ok is False + assert "answer_correct=true" in incomplete_note + assert "provenance=trace incomplete" in incomplete_note + assert "sentinel" not in incomplete_note + + # The real canary supplies this exact empty trace. Every affected read verifier + # must reject even when its text happens to be correct. + empty_run = { + "final_text": "", + "calls": [], + "call_source": "canary", + "evidence_trace_available": False, + } + cases = [ + ("R1", verify_r1, {"r1_state_name": "Investigating 4821"}, "state: Investigating 4821"), + ("R2", verify_r2, {"r2_urgent_open_count": 6}, "count: 6"), + ("R3", verify_r3, {"r3_due_titles": ["Due case 4821"]}, "item: Due case 4821"), + ( + "R4", + verify_r4, + { + "r4_cycle_name": "Sprint 47", + "r4_active_titles": ["Active case 4821"], + "r4_overdue_titles": ["Active case 4821"], + }, + "cycle: Sprint 47\nitem: Active case 4821\noverdue: Active case 4821", + ), + ("R5", verify_r5, {"r5_comment_phrases": ["comment ref-4821"]}, "comment: comment ref-4821"), + ("R6", verify_r6, {"r6_more_bugs_project": "EVAL deadbeef"}, "project: EVAL deadbeef"), + ("I2", verify_i2, {"i2_state_name": "Investigating 4821"}, "state: Investigating 4821"), + ("L2", verify_l2, {"l2_activity_count": 3}, "count: 3"), + ("L5", verify_l5, {"l5_attachment_count": 2}, "count: 2"), + ] + for task_id, verifier, task_ctx, correct_text in cases: + ok, note = await verifier(object(), task_ctx, {**empty_run, "final_text": correct_text}) + assert ok is False, f"{task_id}: {note}" + assert "answer_correct=true" in note, f"{task_id}: {note}" + assert "provenance=unavailable" in note, f"{task_id}: {note}" + + return asyncio.run(_go()) + + +def test_r7_state_group_contract_matches_live_api_exactly(): async def _go(): states = [ - SimpleNamespace(name="Backlog"), - SimpleNamespace(name="In Progress"), - SimpleNamespace(name="Done"), + SimpleNamespace(name="Backlog", group="backlog"), + SimpleNamespace(name="In Progress", group="started"), + SimpleNamespace(name="Done", group="completed"), ] plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(states))) - ctx = {"workspace_slug": "ws", "project_id": "project"} + ctx = { + "workspace_slug": "ws", + "project_id": "project", + "r7_state_pairs": [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + ], + } - exact_ok, note = await verify_r7(plane, ctx, _run("transition: Done")) + exact = "\n".join( + [ + "state: Done | group: completed", + "state: Backlog | group: backlog", + "state: In Progress | group: started", + ] + ) + exact_ok, note = await verify_r7(plane, ctx, _run(exact)) + unrestricted_ok, _ = await verify_r7(plane, ctx, _run("state: unrestricted")) + wrong_group_ok, _ = await verify_r7( + plane, + ctx, + _run(exact.replace("Done | group: completed", "Done | group: started")), + ) + names_only_ok, _ = await verify_r7(plane, ctx, _run("state: Backlog\nstate: In Progress\nstate: Done")) prose_ok, _ = await verify_r7(plane, ctx, _run("It can move to Done.")) + empty_ok, _ = await verify_r7(plane, ctx, _run()) assert exact_ok is True, note + assert unrestricted_ok is False + assert wrong_group_ok is False + assert names_only_ok is False assert prose_ok is False + assert empty_ok is False + + return asyncio.run(_go()) + + +def test_r7_rejects_oracle_mutation_and_zero_call_default_state_cans(): + async def _go(): + baseline = [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + "Review 7b0a1f9c | group: started", + ] + ctx = {"workspace_slug": "ws", "project_id": "project", "r7_state_pairs": baseline} + mutated_states = [ + SimpleNamespace(name="Backlog", group="backlog"), + SimpleNamespace(name="In Progress", group="started"), + SimpleNamespace(name="Done", group="completed"), + SimpleNamespace(name="Agent Rewrite", group="completed"), + ] + mutated_answer = "\n".join( + f"state: {value}" + for value in [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + "Agent Rewrite | group: completed", + ] + ) + mutated_ok, mutated_note = await verify_r7( + SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(mutated_states))), + ctx, + _run(mutated_answer), + ) + assert mutated_ok is False + assert "oracle was mutated after seeding" in mutated_note + + live_baseline = [ + SimpleNamespace(name=value.split(" | group: ")[0], group=value.split(" | group: ")[1]) for value in baseline + ] + canned = "state: Backlog | group: backlog\nstate: In Progress | group: started\nstate: Done | group: completed" + canned_ok, canned_note = await verify_r7( + SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(live_baseline))), + ctx, + _run(canned, calls=[]), + ) + assert canned_ok is False + assert "answer_correct=false" in canned_note + assert "provenance=missing" in canned_note return asyncio.run(_go()) @@ -201,13 +415,14 @@ async def _go(): return asyncio.run(_go()) -def test_w2_requires_the_done_group_specifically(): - """Cancelled is also terminal, so a verifier keying on "not started" would pass it.""" +def test_w2_requires_the_exact_done_state(): + """Other terminal and completed-group states are not the requested Done state.""" async def _go(): cases = [ ("untouched: still in progress", "started", "In Progress"), ("cancelled, not done", "cancelled", "Cancelled"), + ("different completed-group state", "completed", "Closed"), ] for label, group, name in cases: ok, note = await verify_w2(_W2Plane(group, name), {"workspace_slug": "ws", "project_id": "p1"}, _run()) @@ -219,10 +434,18 @@ async def _go(): def test_w4_requires_the_label_renamed_to_the_exact_target(): async def _go(): ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - for label, name in [("untouched: still triage", "triage"), ("renamed to something else", "needs-review")]: + for label, name in [ + ("untouched: still triage", "triage"), + ("renamed to something else", "needs-review"), + ("space is not the requested hyphen", "needs triage"), + ]: ok, note = await verify_w4(_W4Plane(name), dict(ctx), _run()) assert ok is False, f"{label}: {note}" + fallback_ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w4(_W4Plane("needs triage"), fallback_ctx, _run()) + assert ok is False, f"name-scan fallback accepted a space-separated label: {note}" + return asyncio.run(_go()) @@ -249,8 +472,85 @@ async def _go(): True, ), ] + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": CHANGELOG, + } + plane = _C2Plane(f"

{CHANGELOG}

") for label, text, want in cases: - ok, note = await verify_c2(object(), {"release_changelog_text": CHANGELOG}, _run(text)) + ok, note = await verify_c2(plane, ctx, _run(text)) assert ok is want, f"{label}: {note}" return asyncio.run(_go()) + + +def test_c2_live_changelog_behaviours(): + mutated = "Changelog entry one: Live API fact. Changelog entry two: Different live item." + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": CHANGELOG, + } + baseline_answer = "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff" + + async def _go(): + identical_ok, identical_note = await verify_c2(_C2Plane(f"

{CHANGELOG}

"), ctx, _run(baseline_answer)) + assert identical_ok is True, identical_note + + mutated_ok, mutated_note = await verify_c2(_C2Plane(f"

{mutated}

"), ctx, _run(baseline_answer)) + assert mutated_ok is False + assert "changelog was mutated after seeding" in mutated_note + + live_answer = "release: 1.2.0\nshipped: Live API fact\nshipped: Different live item" + exploit_ok, exploit_note = await verify_c2(_C2Plane(f"

{mutated}

"), ctx, _run(live_answer)) + assert exploit_ok is False, exploit_note + + renamed_ok, renamed_note = await verify_c2( + _C2Plane(f"

{CHANGELOG}

", release_name="9.9.9-agent"), + ctx, + _run(baseline_answer), + ) + assert renamed_ok is False + assert "release name was mutated after seeding" in renamed_note + + empty_live_ok, empty_live_note = await verify_c2(_C2Plane("

"), ctx, _run(baseline_answer)) + assert empty_live_ok is False + assert "mutated after seeding" in empty_live_note + assert "live changelog is empty" in empty_live_note + + empty_seed_ctx = {**ctx, "release_changelog_text": ""} + empty_seed_ok, empty_seed_note = await verify_c2(_C2Plane("

"), empty_seed_ctx, _run()) + assert empty_seed_ok is False + assert "fixture missing" in empty_seed_note + assert "seeded changelog baseline is empty" in empty_seed_note + + with pytest.raises(RuntimeError, match="C2 verifier read failed while reading release"): + await verify_c2(_C2Plane(error=RuntimeError("503 unavailable")), ctx, _run()) + + return asyncio.run(_go()) + + +def test_c2_repository_constant_answer_without_reading_fails(): + randomized = ( + "Changelog entry one: OAuth login hardening ticket EVAL-a91c7e20. " + "Changelog entry two: webhook retry backoff window 7-a91c7e20." + ) + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-random", "name": "1.8.14-eval.a91c7e20"}, + "release_changelog_text": randomized, + } + repository_constant_answer = "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff" + + ok, note = asyncio.run( + verify_c2( + _C2Plane(f"

{randomized}

", release_name="1.8.14-eval.a91c7e20"), + ctx, + _run(repository_constant_answer, calls=[]), + ) + ) + + assert ok is False + assert "answer_correct=false" in note + assert "provenance=missing" in note diff --git a/tests/evals/tasks/test_pagination.py b/tests/evals/tasks/test_pagination.py new file mode 100644 index 00000000..30109a32 --- /dev/null +++ b/tests/evals/tasks/test_pagination.py @@ -0,0 +1,142 @@ +"""Regression tests for verifier reads whose target can land after page one.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from plane.errors.errors import HttpError + +from evals.seed import CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, R1_TITLE +from evals.tasks.cross import verify_c1 +from evals.tasks.debias import L3_TAG_VERSION, L4_PROP_DISPLAY, L4_PROP_VALUE, verify_l3, verify_l4 +from evals.tasks.write import W10_PAGE_BODY, W10_PAGE_NAME, verify_w5, verify_w10 + + +class _Page: + def __init__(self, results: list[Any], *, more: bool, cursor: str = ""): + self.results = results + self.next_page_results = more + self.next_cursor = cursor + + +def _cursor(params: Any) -> str | None: + if isinstance(params, dict): + return params.get("cursor") + return getattr(params, "cursor", None) + + +class _TwoPageList: + def __init__(self, target: Any): + self.target = target + self.cursors: list[str | None] = [] + + def list(self, *, params=None, **kwargs): + cursor = _cursor(params) + self.cursors.append(cursor) + if cursor is None: + return _Page([], more=True, cursor="cursor-2") + assert cursor == "cursor-2" + return _Page([self.target], more=False) + + +def test_c1_paginates_customers_requests_and_customer_work_items(): + customer = SimpleNamespace(id="customer-1", name=CUSTOMER_NAME) + request = SimpleNamespace(id="request-1", name=CUSTOMER_REQUEST_NAME) + linked = SimpleNamespace(id="wi-r1") + customers = _TwoPageList(customer) + requests = _TwoPageList(request) + customer_work_items = _TwoPageList(linked) + project_work_items = SimpleNamespace( + list=lambda **kwargs: _Page( + [SimpleNamespace(id="wi-r1", name=R1_TITLE, created_at="2026-01-01")], + more=False, + ) + ) + plane = SimpleNamespace( + work_items=project_work_items, + customers=SimpleNamespace( + list=customers.list, + requests=SimpleNamespace(list=requests.list), + work_items=SimpleNamespace(list=customer_work_items.list), + ), + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "workspace_objects": []} + + ok, note = asyncio.run(verify_c1(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert customers.cursors == [None, "cursor-2"] + assert requests.cursors == [None, "cursor-2"] + assert customer_work_items.cursors == [None, "cursor-2"] + + +def test_l3_paginates_release_tags(): + tags = _TwoPageList(SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)) + plane = SimpleNamespace(releases=SimpleNamespace(tags=SimpleNamespace(list=tags.list))) + ctx = {"workspace_slug": "ws", "workspace_objects": []} + + ok, note = asyncio.run(verify_l3(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert tags.cursors == [None, "cursor-2"] + + +def test_l4_paginates_customer_properties(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, property_type="TEXT") + props = _TwoPageList(prop) + plane = SimpleNamespace( + customers=SimpleNamespace( + properties=SimpleNamespace(list=props.list), + property_values=SimpleNamespace(list=lambda **kwargs: {"prop-1": [L4_PROP_VALUE]}), + ) + ) + ctx = { + "workspace_slug": "ws", + "customer": {"id": "customer-1"}, + "workspace_objects": [], + } + + ok, note = asyncio.run(verify_l4(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert props.cursors == [None, "cursor-2"] + + +def test_w5_paginates_archived_work_items(): + archived = _TwoPageList(SimpleNamespace(id="wi-archived")) + + def missing(**kwargs): + raise HttpError("not found", status_code=404, response={}) + + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=missing, + list_archived=archived.list, + ) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "module_completed_ids": ["wi-archived"], + } + + ok, note = asyncio.run(verify_w5(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert archived.cursors == [None, "cursor-2"] + + +def test_w10_paginates_pages_then_retrieves_the_page_two_body(): + pages = _TwoPageList(SimpleNamespace(id="page-1", name=W10_PAGE_NAME)) + plane = SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=pages.list, + retrieve_project_page=lambda **kwargs: SimpleNamespace( + id="page-1", + description_html=f"

{W10_PAGE_BODY}

", + ), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + + ok, note = asyncio.run(verify_w10(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert pages.cursors == [None, "cursor-2"] diff --git a/tests/evals/tasks/test_verifier_read_errors.py b/tests/evals/tasks/test_verifier_read_errors.py new file mode 100644 index 00000000..128486c9 --- /dev/null +++ b/tests/evals/tasks/test_verifier_read_errors.py @@ -0,0 +1,510 @@ +"""Verifier API-read failures are infrastructure; explicit fallbacks stay tolerant.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from datetime import date +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.changelog import normalize_changelog_text +from evals.fixtures import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + CYCLE_CURRENT, + INTAKE_BILLING_TITLE, + INTAKE_SPAM_TITLE, + R1_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, +) +from evals.tasks.cross import verify_c1, verify_c2 +from evals.tasks.debias import L1_TITLE, L4_PROP_DISPLAY, verify_l1, verify_l3, verify_l4 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.verification import VerifierReadError +from evals.tasks.write import ( + W10_PAGE_NAME, + W11_TITLE, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w10, + verify_w11, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _http(status: int) -> HttpError: + return HttpError("read unavailable" if status != 404 else "not found", status, {}) + + +def _raise(exc: BaseException) -> Callable[..., Any]: + def fail(**kwargs: Any) -> Any: + raise exc + + return fail + + +def _run() -> dict[str, Any]: + return {"final_text": "", "calls": []} + + +def _s1(site: str) -> tuple[Any, dict[str, Any]]: + severity = SimpleNamespace(id="severity-1", display_name="Severity", property_type="OPTION", options=[]) + if site == "properties": + properties = SimpleNamespace(list=_raise(_http(500))) + else: + properties = SimpleNamespace(list=lambda **kw: [severity], options=SimpleNamespace(list=_raise(_http(500)))) + return SimpleNamespace(work_item_properties=properties), { + "workspace_slug": "ws", + "project_id": "p1", + "bug_type": {"id": "bug-1"}, + } + + +def _s3(site: str) -> tuple[Any, dict[str, Any]]: + incident = SimpleNamespace(id="incident-1", name="Incident") + project_types = [] if site in {"ownership", "workspace-types"} else [incident] + workspace_features = ( + _raise(ConnectionError("features down")) + if site == "ownership" + else lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ) + workspace_types = _raise(ConnectionError("types down")) if site == "workspace-types" else lambda **kw: [] + property_list = _raise(_http(500)) if site == "properties" else lambda **kw: [] + return SimpleNamespace( + work_item_types=SimpleNamespace(list=lambda **kw: project_types), + workspaces=SimpleNamespace(get_features=workspace_features), + workspace_work_item_types=SimpleNamespace(list=workspace_types), + work_item_properties=SimpleNamespace(list=property_list), + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _s5(site: str) -> tuple[Any, dict[str, Any]]: + project_features = ( + _raise(ConnectionError("project features down")) + if site == "project" + else lambda **kw: SimpleNamespace(model_dump=lambda: {"cycles": True}) + ) + workspace_features = _raise(ConnectionError("workspace features down")) + return SimpleNamespace( + projects=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(cycle_view=True, is_time_tracking_enabled=True), + get_features=project_features, + ), + workspaces=SimpleNamespace(get_features=workspace_features), + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _l4(site: str) -> tuple[Any, dict[str, Any]]: + prop = SimpleNamespace(id="property-1", display_name=L4_PROP_DISPLAY, property_type="TEXT") + properties = _raise(ConnectionError("properties down")) if site == "properties" else lambda **kw: _Page([prop]) + values = _raise(ConnectionError("values down")) + return SimpleNamespace( + customers=SimpleNamespace( + properties=SimpleNamespace(list=properties), + property_values=SimpleNamespace(list=values), + ) + ), {"workspace_slug": "ws", "customer": {"id": "customer-1"}} + + +def _c1() -> tuple[Any, dict[str, Any]]: + r1 = SimpleNamespace(id="item-r1", name=R1_TITLE, created_at="2026-01-01") + customer = SimpleNamespace(id="customer-1", name=CUSTOMER_NAME) + request = SimpleNamespace(id="request-1", name=CUSTOMER_REQUEST_NAME) + return SimpleNamespace( + work_items=SimpleNamespace(list=lambda **kw: _Page([r1])), + customers=SimpleNamespace( + list=lambda **kw: _Page([customer]), + requests=SimpleNamespace(list=lambda **kw: _Page([request])), + work_items=SimpleNamespace(list=_raise(ConnectionError("links down"))), + ), + ), {"workspace_slug": "ws", "project_id": "p1", "workspace_objects": []} + + +def _w5(site: str) -> tuple[Any, dict[str, Any]]: + retrieve = _raise(_http(500) if site == "retrieve" else _http(404)) + return SimpleNamespace( + work_items=SimpleNamespace( + retrieve=retrieve, + list_archived=_raise(ConnectionError("archive list down")), + ) + ), {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + + +def _w7(site: str) -> tuple[Any, dict[str, Any]]: + rows = [ + SimpleNamespace(id="source-1", name=W7_SOURCE_TITLE, created_at="2026-01-02"), + SimpleNamespace(id="target-1", name=W7_TARGET_TITLE, created_at="2026-01-01"), + ] + dependencies = ( + _raise(ConnectionError("dependencies down")) + if site == "dependencies" + else lambda **kw: {"blocking": [{"id": "target-1"}]} + ) + links = _raise(ConnectionError("links down")) + return SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page(rows), + dependencies=SimpleNamespace(list=dependencies), + links=SimpleNamespace(list=links), + ) + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _w10(site: str) -> tuple[Any, dict[str, Any]]: + page = SimpleNamespace(id="page-1", name=W10_PAGE_NAME) + listing = _raise(ConnectionError("pages down")) if site == "list" else lambda **kw: _Page([page]) + return SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=listing, + retrieve_project_page=_raise(TimeoutError("page read timed out")), + ) + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _infra_cases() -> list[Any]: + cases: list[Any] = [] + + def add(case_id: str, task: str, reading: str, verifier: Any, plane: Any, ctx: dict[str, Any]) -> None: + cases.append(pytest.param(task, reading, lambda: verifier(plane, ctx, _run()), id=case_id)) + + plane, ctx = _s1("properties") + add("S1-type-properties", "S1", "listing Bug type properties", verify_s1, plane, ctx) + plane, ctx = _s1("options") + add("S1-options", "S1", "listing Severity options", verify_s1, plane, ctx) + add( + "S2-estimate", + "S2", + "retrieving the project estimate", + verify_s2, + SimpleNamespace(estimates=SimpleNamespace(retrieve=_raise(_http(500)))), + {"workspace_slug": "ws", "project_id": "p1"}, + ) + for site, reading in ( + ("ownership", "reading workspace work-item-type ownership"), + ("workspace-types", "listing workspace work-item types"), + ("properties", "listing Incident type properties"), + ): + plane, ctx = _s3(site) + add(f"S3-{site}", "S3", reading, verify_s3, plane, ctx) + add( + "S4-intake-list-fallback", + "S4", + "listing intake while resolving", + verify_s4, + SimpleNamespace( + intake=SimpleNamespace( + retrieve=_raise(ConnectionError("retrieve down")), + list=_raise(ConnectionError("list down")), + ) + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": "billing-1"}, "spam": {"issue_id": "spam-1"}}, + }, + ) + for site, reading in (("project", "reading project feature flags"), ("workspace", "reading workspace customer")): + plane, ctx = _s5(site) + add(f"S5-{site}-features", "S5", reading, verify_s5, plane, ctx) + add( + "L1-worklog-summary", + "L1", + "reading the project worklog summary", + verify_l1, + SimpleNamespace( + work_items=SimpleNamespace(work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=90)])), + projects=SimpleNamespace(get_worklog_summary=_raise(ConnectionError("summary down"))), + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L1_TITLE: "item-1"}, + "l1_expected_summary_ids": ["item-1"], + }, + ) + add( + "L3-release-tags", + "L3", + "listing workspace release tags", + verify_l3, + SimpleNamespace(releases=SimpleNamespace(tags=SimpleNamespace(list=_raise(ConnectionError("tags down"))))), + {"workspace_slug": "ws"}, + ) + for site, reading in ( + ("properties", "listing workspace customer properties"), + ("values", "reading property values"), + ): + plane, ctx = _l4(site) + add(f"L4-{site}", "L4", reading, verify_l4, plane, ctx) + plane, ctx = _c1() + add("C1-customer-work-items", "C1", "listing work items linked to customer", verify_c1, plane, ctx) + baseline = "Changelog entry one: One. Changelog entry two: Two." + add( + "C2-release-changelog", + "C2", + "reading release release-1 and its changelog", + verify_c2, + SimpleNamespace( + releases=SimpleNamespace( + retrieve=_raise(_http(500)), + changelog=SimpleNamespace(retrieve=lambda **kw: normalize_changelog_text(baseline)), + ) + ), + { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": baseline, + }, + ) + add( + "W4-triage-label", + "W4", + "retrieving seeded triage label", + verify_w4, + SimpleNamespace(labels=SimpleNamespace(retrieve=_raise(_http(500)))), + {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "label-1"}}, + ) + for site, reading in (("retrieve", "retrieving module work item"), ("archived", "listing archived items")): + plane, ctx = _w5(site) + add(f"W5-{site}", "W5", reading, verify_w5, plane, ctx) + add( + "W6-cycle-items", + "W6", + f"listing {CYCLE_CURRENT} work items", + verify_w6, + SimpleNamespace( + cycles=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + end_date=date.today().isoformat(), archived_at=None, progress_snapshot=None + ), + list_work_items=_raise(TimeoutError("cycle items read timed out")), + ) + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "past-1", + "cycle_current_id": "current-1", + "w6_unfinished_titles": ["unfinished"], + }, + ) + for site, reading in (("dependencies", "listing dependencies"), ("links", "listing links")): + plane, ctx = _w7(site) + add(f"W7-{site}", "W7", reading, verify_w7, plane, ctx) + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + add( + "W11-worklogs", + "W11", + "listing work logs for item", + verify_w11, + SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page([item]), + work_logs=SimpleNamespace(list=_raise(_http(500))), + ) + ), + {"workspace_slug": "ws", "project_id": "p1"}, + ) + for site, reading in (("list", "listing project pages"), ("retrieve", "retrieving project page")): + plane, ctx = _w10(site) + add(f"W10-{site}", "W10", reading, verify_w10, plane, ctx) + return cases + + +@pytest.mark.parametrize(("task_id", "reading", "invoke"), _infra_cases()) +def test_required_verifier_read_failures_are_infrastructure(task_id: str, reading: str, invoke: Callable[[], Any]): + with pytest.raises(VerifierReadError) as caught: + asyncio.run(invoke()) + + message = str(caught.value) + assert message.startswith(f"{task_id} verifier read failed while ") + assert reading in message + assert caught.value.__cause__ is not None + + +async def _negative_s2_estimate_404() -> tuple[bool, str]: + plane = SimpleNamespace(estimates=SimpleNamespace(retrieve=_raise(_http(404)))) + return await verify_s2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _negative_c2_oracle_404() -> tuple[bool, str]: + baseline = "Changelog entry one: One. Changelog entry two: Two." + plane = SimpleNamespace( + releases=SimpleNamespace( + retrieve=_raise(_http(404)), + changelog=SimpleNamespace(retrieve=lambda **kw: normalize_changelog_text(baseline)), + ) + ) + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": baseline, + } + return await verify_c2(plane, ctx, _run()) + + +async def _negative_w6_current_cycle_404() -> tuple[bool, str]: + plane = SimpleNamespace( + cycles=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + end_date=date.today().isoformat(), archived_at=None, progress_snapshot=None + ), + list_work_items=_raise(_http(404)), + ) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "past-1", + "cycle_current_id": "current-1", + "w6_unfinished_titles": ["unfinished"], + } + return await verify_w6(plane, ctx, _run()) + + +async def _negative_w10_page_retrieve_404() -> tuple[bool, str]: + page = SimpleNamespace(id="page-1", name=W10_PAGE_NAME) + plane = SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=lambda **kw: _Page([page]), + retrieve_project_page=_raise(_http(404)), + ) + ) + return await verify_w10(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +@pytest.mark.parametrize( + ("invoke", "note_fragment"), + [ + pytest.param(_negative_s2_estimate_404, "estimate not found", id="S2-missing-created-estimate"), + pytest.param(_negative_c2_oracle_404, "no longer exists", id="C2-missing-seed-oracle"), + pytest.param(_negative_w6_current_cycle_404, "not found", id="W6-missing-rollover-cycle"), + pytest.param(_negative_w10_page_retrieve_404, "not found after listing", id="W10-missing-created-page"), + ], +) +def test_authoritative_not_found_is_an_agent_failure(invoke: Callable[[], Any], note_fragment: str): + success, note = asyncio.run(invoke()) + assert success is False + assert note_fragment in note + + +async def _tolerant_s1_property_404() -> tuple[bool, str]: + plane = SimpleNamespace(work_item_properties=SimpleNamespace(list=_raise(_http(404)))) + return await verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug"}}, _run()) + + +async def _tolerant_s1_options_404() -> tuple[bool, str]: + severity = SimpleNamespace(id="severity", display_name="Severity", property_type="OPTION", options=[]) + plane = SimpleNamespace( + work_item_properties=SimpleNamespace( + list=lambda **kw: [severity], + options=SimpleNamespace(list=_raise(_http(404))), + ) + ) + return await verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug"}}, _run()) + + +async def _tolerant_s3_property_404() -> tuple[bool, str]: + incident = SimpleNamespace(id="incident", name="Incident") + plane = SimpleNamespace( + work_item_types=SimpleNamespace(list=lambda **kw: [incident]), + work_item_properties=SimpleNamespace(list=_raise(_http(404))), + ) + return await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _tolerant_s4_retrieve_fallback() -> tuple[bool, str]: + rows = [ + SimpleNamespace(issue_detail=SimpleNamespace(name=INTAKE_BILLING_TITLE), status=1), + SimpleNamespace(issue_detail=SimpleNamespace(name=INTAKE_SPAM_TITLE), status=-1), + ] + plane = SimpleNamespace( + intake=SimpleNamespace(retrieve=_raise(ConnectionError("retrieve down")), list=lambda **kw: rows) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": "billing"}, "spam": {"issue_id": "spam"}}, + } + return await verify_s4(plane, ctx, _run()) + + +async def _tolerant_w4_not_found() -> tuple[bool, str]: + plane = SimpleNamespace(labels=SimpleNamespace(retrieve=_raise(_http(404)))) + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "label-1"}} + return await verify_w4(plane, ctx, _run()) + + +async def _tolerant_w5_retrieve_fallback() -> tuple[bool, str]: + archived = SimpleNamespace(id="item-1") + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=_raise(_http(404)), + list_archived=lambda **kw: _Page([archived]), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + return await verify_w5(plane, ctx, _run()) + + +async def _tolerant_w5_optional_archive_crosscheck() -> tuple[bool, str]: + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(id="item-1", archived_at=None), + list_archived=_raise(ConnectionError("optional list down")), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + return await verify_w5(plane, ctx, _run()) + + +async def _tolerant_w11_diagnostic_read() -> tuple[bool, str]: + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + plane = SimpleNamespace( + work_items=SimpleNamespace(list=lambda **kw: _Page([item]), work_logs=SimpleNamespace(list=lambda **kw: [])), + projects=SimpleNamespace(retrieve=_raise(ConnectionError("diagnostic read down"))), + ) + return await verify_w11(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _tolerant_w11_gate_404() -> tuple[bool, str]: + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + plane = SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page([item]), + work_logs=SimpleNamespace(list=_raise(_http(404))), + ) + ) + return await verify_w11(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +@pytest.mark.parametrize( + ("invoke", "want_success"), + [ + pytest.param(_tolerant_s1_property_404, False, id="S1-property-404-is-absence"), + pytest.param(_tolerant_s1_options_404, False, id="S1-options-404-is-absence"), + pytest.param(_tolerant_s3_property_404, False, id="S3-property-404-is-absence"), + pytest.param(_tolerant_s4_retrieve_fallback, True, id="S4-retrieve-has-list-fallback"), + pytest.param(_tolerant_w4_not_found, False, id="W4-seed-id-404-is-deletion"), + pytest.param(_tolerant_w5_retrieve_fallback, True, id="W5-retrieve-404-has-archive-fallback"), + pytest.param(_tolerant_w5_optional_archive_crosscheck, False, id="W5-archive-list-crosscheck-is-optional"), + pytest.param(_tolerant_w11_diagnostic_read, False, id="W11-feature-read-is-diagnostic-only"), + pytest.param(_tolerant_w11_gate_404, False, id="W11-worklog-404-is-disabled-gate"), + ], +) +def test_deliberately_tolerant_verifier_reads_are_pinned(invoke: Callable[[], Any], want_success: bool): + success, note = asyncio.run(invoke()) + assert success is want_success, note diff --git a/tests/evals/tasks/test_verifiers.py b/tests/evals/tasks/test_verifiers.py index a944487d..57916fd7 100644 --- a/tests/evals/tasks/test_verifiers.py +++ b/tests/evals/tasks/test_verifiers.py @@ -7,8 +7,10 @@ from types import SimpleNamespace from typing import Any +import pytest from plane.errors.errors import HttpError +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, @@ -23,7 +25,17 @@ from evals.tasks.cross import verify_c1 from evals.tasks.read import verify_r3 from evals.tasks.schema import verify_s3, verify_s5 -from evals.tasks.write import verify_w3, verify_w4, verify_w5, verify_w6, verify_w7, verify_w8 +from evals.tasks.write import ( + W10_PAGE_BODY, + W10_PAGE_NAME, + verify_w3, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w8, + verify_w10, +) class _Page: @@ -72,47 +84,52 @@ def _links_list(self, **kw): return _Page([SimpleNamespace(url=u) for u in self._urls]) -def test_f1_w7_behaviours(): - def test_f1_w7_blocked_by_only_does_not_pass(): - async def _go(): - """tgt id only in blocked_by (reverse) must FAIL — not pass via str(dump).""" - plane = _W7Plane( - { - "blocking": [], - "blocked_by": [{"id": "tgt-1"}], # reverse direction only - } - ) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_w7(plane, ctx, _run()) - assert ok is False, note - assert "blocking" in note.lower() or "no blocking" in note.lower() - assert "wrong direction" in note or "tgt-1" in note - - return asyncio.run(_go()) - - def test_f1_w7_blocking_passes(): - async def _go(): - plane = _W7Plane({"blocking": [{"id": "tgt-1"}], "blocked_by": []}) - ctx = {"workspace_slug": "ws", "project_id": "p1"} - ok, note = await verify_w7(plane, ctx, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F2 W6 — seeded past end_date alone must NOT pass as closed - # --------------------------------------------------------------------------- - - return asyncio.run(_go()) - - test_f1_w7_blocked_by_only_does_not_pass() - test_f1_w7_blocking_passes() +@pytest.mark.parametrize( + ("dependencies", "want", "expect_wrong_direction"), + [ + pytest.param( + {"blocking": [], "blocked_by": [{"id": "tgt-1"}]}, + False, + True, + id="blocked-by-is-wrong-direction", + ), + pytest.param( + {"blocking": [{"id": "tgt-1"}], "blocked_by": []}, + True, + False, + id="blocking-passes", + ), + ], +) +def test_f1_w7_behaviours(dependencies, want, expect_wrong_direction): + ok, note = asyncio.run( + verify_w7( + _W7Plane(dependencies), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect_wrong_direction: + assert "blocking" in note.lower() or "no blocking" in note.lower() + assert "wrong direction" in note or "tgt-1" in note class _W6Plane: - def __init__(self, *, past_end: str, archived_at=None, snapshot=None, sprint13_names: list[str] | None = None): + def __init__( + self, + *, + past_end: str, + archived_at=None, + snapshot=None, + sprint13_names: list[str] | None = None, + list_error: Exception | None = None, + ): self._past_end = past_end self._archived_at = archived_at self._snapshot = snapshot self._s13 = sprint13_names or [] + self._list_error = list_error self.cycles = SimpleNamespace(retrieve=self._retrieve, list_work_items=self._list_wi) def _retrieve(self, **kw): @@ -124,42 +141,120 @@ def _retrieve(self, **kw): ) def _list_wi(self, **kw): + if self._list_error is not None: + raise self._list_error return _Page([_item(f"i{i}", n) for i, n in enumerate(self._s13)]) -def test_f2_w6_closes_only_on_a_real_end_date_signal(): - """The API returns end_date as a timestamp, so whole-string comparison to today never - matched — which silently left the transfer side effect as the only way to pass.""" - - async def _go(): - today = date.today().isoformat() - past = (date.today() - timedelta(days=14)).isoformat() - tomorrow = (date.today() + timedelta(days=1)).isoformat() - titles = ["Inventory count goes negative under load"] - two = [*titles, "Tooltip clipped inside modal dialog"] - cases = [ - ("still at the seeded past end, never closed", past, past, titles, False, "not closed"), - ("no-op agent: still ends tomorrow", f"{tomorrow}T00:00:00Z", tomorrow, titles, False, "not closed"), - ("closed today, returned as a timestamp", f"{today}T00:00:00Z", tomorrow, titles, True, "end_date"), - ("closed today, returned as a bare date", today, past, two, True, ""), - ] - for label, end_date, seed_end, names, want, expect in cases: - plane = _W6Plane(past_end=end_date, sprint13_names=names) - ctx = { - "workspace_slug": "ws", - "project_id": "p1", - "cycle_past_id": "c12", - "cycle_current_id": "c13", - "cycle_past_seed_end_date": seed_end, - "w6_unfinished_titles": names, - "cycles": {CYCLE_PAST: "c12"}, - } - ok, note = await verify_w6(plane, ctx, _run()) - assert ok is want, f"{label}: {note}" - if expect: - assert expect in note.lower() or expect in note, f"{label}: {note}" - - return asyncio.run(_go()) +@pytest.mark.parametrize( + ("end_offset", "timestamp", "seed_offset", "names", "want", "expect"), + [ + pytest.param( + -14, False, -14, ["Inventory count goes negative under load"], False, "not closed", id="seeded-past-end" + ), + pytest.param(1, True, 1, ["Inventory count goes negative under load"], False, "not closed", id="noop-tomorrow"), + pytest.param( + 0, True, 1, ["Inventory count goes negative under load"], True, "end_date", id="closed-today-timestamp" + ), + pytest.param( + 0, + False, + -14, + ["Inventory count goes negative under load", "Tooltip clipped inside modal dialog"], + True, + "", + id="closed-today-date", + ), + ], +) +def test_f2_w6_closes_only_on_a_real_end_date_signal(end_offset, timestamp, seed_offset, names, want, expect): + """A timestamp and a bare date must provide the same real closure signal.""" + end_date = (date.today() + timedelta(days=end_offset)).isoformat() + if timestamp: + end_date = f"{end_date}T00:00:00Z" + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() + timedelta(days=seed_offset)).isoformat(), + "w6_unfinished_titles": names, + "cycles": {CYCLE_PAST: "c12"}, + } + ok, note = asyncio.run(verify_w6(_W6Plane(past_end=end_date, sprint13_names=names), ctx, _run())) + assert ok is want, note + if expect: + assert expect in note.lower() or expect in note, note + + +@pytest.mark.parametrize( + ("sprint13_names", "list_error", "ctx_override", "want", "expect_note", "raises"), + [ + pytest.param(None, None, {}, True, "", None, id="healthy"), + pytest.param( + ["Inventory count goes negative under load"], + None, + {}, + False, + "unfinished not on Sprint 13", + None, + id="missing-rollover-item", + ), + pytest.param(None, RuntimeError("503 unavailable"), {}, None, "", "W6.*Sprint 13", id="listing-error"), + pytest.param( + None, + None, + {"cycle_current_id": None}, + None, + "", + "W6 fixture error.*Sprint 13 id missing", + id="missing-current-cycle", + ), + pytest.param( + None, + None, + {"cycle_past_id": None}, + None, + "", + "W6 fixture error.*Sprint 12 id missing", + id="missing-past-cycle", + ), + pytest.param( + None, + None, + {"w6_unfinished_titles": []}, + None, + "", + "W6 fixture error.*unfinished items.*empty", + id="empty-unfinished-fixture", + ), + ], +) +def test_w6_rollover_verification_is_fail_closed(sprint13_names, list_error, ctx_override, want, expect_note, raises): + expected = ["Inventory count goes negative under load", "Tooltip clipped inside modal dialog"] + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() + timedelta(days=1)).isoformat(), + "w6_unfinished_titles": expected, + **ctx_override, + } + plane = _W6Plane( + past_end=date.today().isoformat(), + sprint13_names=expected if sprint13_names is None else sprint13_names, + list_error=list_error, + ) + if raises: + with pytest.raises(RuntimeError, match=raises): + asyncio.run(verify_w6(plane, ctx, _run())) + return + + ok, note = asyncio.run(verify_w6(plane, ctx, _run())) + assert ok is want, note + if expect_note: + assert expect_note in note class _W5Plane: @@ -185,31 +280,38 @@ def _list_archived(self, **kw): return _Page([_item(i, f"n-{i}") for i in self._archived_ids]) -def test_f3_w5_accepts_archive_but_not_deletion(): - """A 404 alone is not evidence of archiving — deleting every item would also 404.""" - - async def _go(): - ids = ["m1", "m2", "m3"] - archived_at = SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z") - cases = [ - ("all 404, nothing archived", _W5Plane(retrieve_map={}, archived_ids=[]), ids, False, "not archived"), - ("404 but present in the archived list", _W5Plane(retrieve_map={}, archived_ids=ids), ids, True, ""), - ( - "archived_at on retrieve", - _W5Plane(retrieve_map={"m1": archived_at}, archived_ids=[]), - ["m1"], - True, - "", +@pytest.mark.parametrize( + ("plane", "module_ids", "want", "expect"), + [ + pytest.param( + _W5Plane(retrieve_map={}, archived_ids=[]), + ["m1", "m2", "m3"], + False, + "not archived", + id="deleted-not-archived", + ), + pytest.param( + _W5Plane(retrieve_map={}, archived_ids=["m1", "m2", "m3"]), ["m1", "m2", "m3"], True, "", id="archived-list" + ), + pytest.param( + _W5Plane( + retrieve_map={"m1": SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z")}, + archived_ids=[], ), - ] - for label, plane, module_ids, want, expect in cases: - ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": module_ids} - ok, note = await verify_w5(plane, ctx, _run()) - assert ok is want, f"{label}: {note}" - if expect: - assert expect in note, f"{label}: {note}" - - return asyncio.run(_go()) + ["m1"], + True, + "", + id="archived-at", + ), + ], +) +def test_f3_w5_accepts_archive_but_not_deletion(plane, module_ids, want, expect): + """A 404 alone is not evidence of archiving — deleting every item would also 404.""" + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": module_ids} + ok, note = asyncio.run(verify_w5(plane, ctx, _run())) + assert ok is want, note + if expect: + assert expect in note, note class _C1Plane: @@ -229,46 +331,48 @@ def __init__( self.work_items = SimpleNamespace(list=lambda **kw: _Page(project_items)) -def test_f4_c1_requires_the_named_customer_linked_to_the_named_item(): +@pytest.mark.parametrize( + ("customers", "linked", "project_items", "want", "expect_any"), + [ + pytest.param( + [SimpleNamespace(id="c1", name="Acme Industries")], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + False, + (CUSTOMER_NAME,), + id="lookalike-customer", + ), + pytest.param( + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-other")], + [_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], + False, + ("not linked", "wi-r1"), + id="wrong-linked-item", + ), + pytest.param( + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + True, + (), + id="exact-customer-and-item", + ), + ], +) +def test_f4_c1_requires_the_named_customer_linked_to_the_named_item(customers, linked, project_items, want, expect_any): """Both ends are checked: a lookalike customer name and a link to any other item fail.""" - - async def _go(): - request = SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME) - cases = [ - ( - "customer name is a lookalike", - [SimpleNamespace(id="c1", name="Acme Industries")], - [SimpleNamespace(id="wi-r1")], - [_item("wi-r1", R1_TITLE)], - False, - (CUSTOMER_NAME,), - ), - ( - "linked to a different item", - [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], - [SimpleNamespace(id="wi-other")], - [_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], - False, - ("not linked", "wi-r1"), - ), - ( - "exact customer and item", - [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], - [SimpleNamespace(id="wi-r1")], - [_item("wi-r1", R1_TITLE)], - True, - (), - ), - ] - for label, customers, linked, project_items, want, expect_any in cases: - plane = _C1Plane(customers=customers, requests=[request], linked=linked, project_items=project_items) - ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} - ok, note = await verify_c1(plane, ctx, _run()) - assert ok is want, f"{label}: {note}" - if expect_any: - assert any(s in note for s in expect_any), f"{label}: {note}" - - return asyncio.run(_go()) + plane = _C1Plane( + customers=customers, + requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], + linked=linked, + project_items=project_items, + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} + ok, note = asyncio.run(verify_c1(plane, ctx, _run())) + assert ok is want, note + if expect_any: + assert any(s in note for s in expect_any), note class _W3Plane: @@ -279,21 +383,70 @@ def __init__(self, comments: list[Any]): ) -def test_f5_w3_matches_the_phrase_in_either_comment_field(): - async def _go(): - html = "

Reviewed contrast tokens — needs design pass

" - cases = [ - ("unrelated comment", "

lgtm

", "lgtm", False, "contrast tokens"), - ("phrase present", html, "Reviewed contrast tokens — needs design pass", True, ""), - ] - for label, comment_html, stripped, want, expect in cases: - plane = _W3Plane([SimpleNamespace(comment_html=comment_html, comment_stripped=stripped)]) - ok, note = await verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is want, f"{label}: {note}" - if expect: - assert expect in note, f"{label}: {note}" +@pytest.mark.parametrize( + ("comment_html", "stripped", "want", "expect"), + [ + pytest.param("

lgtm

", "lgtm", False, "contrast tokens", id="unrelated-comment"), + pytest.param( + "

Reviewed contrast tokens — needs design pass

", + "Reviewed contrast tokens — needs design pass", + True, + "", + id="exact-stripped-text", + ), + pytest.param( + "

Reviewed contrast tokens — needs design pass

", + None, + True, + "", + id="normalized-html", + ), + pytest.param( + "

Reviewed contrast tokens — needs design pass and accessibility review

", + "Reviewed contrast tokens — needs design pass and accessibility review", + False, + "exact normalized comment", + id="substring-only", + ), + ], +) +def test_f5_w3_requires_exact_normalized_comment_text(comment_html, stripped, want, expect): + plane = _W3Plane([SimpleNamespace(comment_html=comment_html, comment_stripped=stripped)]) + ok, note = asyncio.run(verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + if expect: + assert expect in note, note - return asyncio.run(_go()) + +class _Pages: + def __init__(self, body: str): + self.body = body + + def list_project_pages(self, **kwargs): + return _Page([SimpleNamespace(id="page-1", name=W10_PAGE_NAME)]) + + def retrieve_project_page(self, **kwargs): + return SimpleNamespace(id="page-1", name=W10_PAGE_NAME, description_html=self.body) + + +@pytest.mark.parametrize( + ("body", "want", "expect"), + [ + pytest.param("

Unrelated runbook body

", False, "body mismatch", id="wrong-body"), + pytest.param(f"

{W10_PAGE_BODY}

", True, "", id="normalized-exact-body"), + ], +) +def test_w10_requires_the_exact_normalized_page_body(body, want, expect): + ok, note = asyncio.run( + verify_w10( + SimpleNamespace(pages=_Pages(body)), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect: + assert expect in note class _W4Plane: @@ -308,34 +461,32 @@ def _retrieve(self, **kw): return self._by_id[lid] -def test_f7_w4_follows_the_seeded_label_id_not_the_name(): +@pytest.mark.parametrize( + ("by_id", "listed", "want", "expect"), + [ + pytest.param( + {"triage-id": SimpleNamespace(id="triage-id", name="triage")}, + [SimpleNamespace(id="triage-id", name="triage"), SimpleNamespace(id="other", name="needs-triage")], + False, + "triage-id", + id="decoy-label", + ), + pytest.param( + {"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, + [SimpleNamespace(id="triage-id", name="needs-triage")], + True, + "", + id="seeded-label-renamed", + ), + ], +) +def test_f7_w4_follows_the_seeded_label_id_not_the_name(by_id, listed, want, expect): """A name scan would accept a different label renamed to the target.""" - - async def _go(): - cases = [ - ( - "decoy label carries the new name", - {"triage-id": SimpleNamespace(id="triage-id", name="triage")}, - [SimpleNamespace(id="triage-id", name="triage"), SimpleNamespace(id="other", name="needs-triage")], - False, - "triage-id", - ), - ( - "the seeded label itself was renamed", - {"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, - [SimpleNamespace(id="triage-id", name="needs-triage")], - True, - "", - ), - ] - for label, by_id, listed, want, expect in cases: - ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} - ok, note = await verify_w4(_W4Plane(by_id=by_id, listed=listed), ctx, _run()) - assert ok is want, f"{label}: {note}" - if expect: - assert expect in note, f"{label}: {note}" - - return asyncio.run(_go()) + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = asyncio.run(verify_w4(_W4Plane(by_id=by_id, listed=listed), ctx, _run())) + assert ok is want, note + if expect: + assert expect in note, note class _S3Plane: @@ -351,48 +502,28 @@ def __init__(self, *, props: list[Any], types: list[Any] | None = None, workspac self.workspace_work_item_types = SimpleNamespace(list=lambda **kw: []) -def test_f6_s3_behaviours(): - def test_f6_s3_required_option_does_not_pass(): - async def _go(): - plane = _S3Plane( - props=[ - SimpleNamespace( - id="p1", - display_name="Severity", - property_type="OPTION", - is_required=True, - ) - ] - ) - ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "TEXT" in note - - return asyncio.run(_go()) - - def test_f6_s3_required_text_passes(): - async def _go(): - plane = _S3Plane( - props=[ - SimpleNamespace( - id="p1", - display_name="Impact summary", - property_type="TEXT", - is_required=True, - ) - ] +@pytest.mark.parametrize( + ("display_name", "property_type", "want", "expect"), + [ + pytest.param("Severity", "OPTION", False, "TEXT", id="required-option-fails"), + pytest.param("Impact summary", "TEXT", True, "", id="required-text-passes"), + ], +) +def test_f6_s3_behaviours(display_name, property_type, want, expect): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name=display_name, + property_type=property_type, + is_required=True, ) - ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # F9 S3 — workspace types found via get_features probe (not seed flag) - # --------------------------------------------------------------------------- - - return asyncio.run(_go()) - - test_f6_s3_required_option_does_not_pass() - test_f6_s3_required_text_passes() + ] + ) + ok, note = asyncio.run(verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + if expect: + assert expect in note def test_f9_s3_workspace_type_via_features_probe(): @@ -417,45 +548,38 @@ async def _go(): ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) assert ok is True, note - # --------------------------------------------------------------------------- - # F8 R3 due date stays in current week (seed helper) - # --------------------------------------------------------------------------- - return asyncio.run(_go()) -def test_f8_behaviours(): - def test_f8_r3_due_date_clamped_to_iso_week(): - for weekday in range(7): # 0=Mon … 6=Sun - # Build a fixed "today" with that weekday relative to a known Monday. - # 2026-08-10 is a Monday. - monday = date(2026, 8, 10) - today = monday + timedelta(days=weekday) - days_to_week_end = 6 - today.weekday() - due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) - # Sunday of that week - week_end = today + timedelta(days=days_to_week_end) - week_start = today - timedelta(days=today.weekday()) - assert week_start <= due <= week_end, f"weekday={weekday} due={due}" - # Specifically: Sat/Sun must not go past Sunday - if weekday >= 5: - assert due <= week_end - assert due == week_end or due == today # Sun→today, Sat→Sun - - def test_f8_seed_r3_due_date_function_matches(): - sat = date(2026, 8, 15) # known Saturday - assert sat.weekday() == 5 - days_to_week_end = 6 - sat.weekday() - due = min(sat + timedelta(days=2), sat + timedelta(days=days_to_week_end)) - assert due == date(2026, 8, 16) # Sunday, not Monday 17 - # Sunday - sun = date(2026, 8, 16) - days_to_week_end = 6 - sun.weekday() - due = min(sun + timedelta(days=2), sun + timedelta(days=days_to_week_end)) - assert due == sun - - test_f8_r3_due_date_clamped_to_iso_week() - test_f8_seed_r3_due_date_function_matches() +@pytest.mark.parametrize( + "weekday", + range(7), + ids=["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"], +) +def test_f8_r3_due_date_clamped_to_iso_week(weekday): + monday = date(2026, 8, 10) + today = monday + timedelta(days=weekday) + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + week_end = today + timedelta(days=days_to_week_end) + week_start = today - timedelta(days=today.weekday()) + assert week_start <= due <= week_end, f"weekday={weekday} due={due}" + if weekday >= 5: + assert due <= week_end + assert due == week_end or due == today + + +@pytest.mark.parametrize( + ("today", "expected"), + [ + pytest.param(date(2026, 8, 15), date(2026, 8, 16), id="saturday-clamps-to-sunday"), + pytest.param(date(2026, 8, 16), date(2026, 8, 16), id="sunday-stays-sunday"), + ], +) +def test_f8_seed_r3_due_date_function_matches(today, expected): + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + assert due == expected class _W8Plane: @@ -466,68 +590,60 @@ def __init__(self, durations: list[int]): ) -def test_minor_behaviours(): - def test_minor_w8_behaviours(): - def test_minor_w8_480_minutes_fails(): - async def _go(): - plane = _W8Plane([480]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, note - assert "120" in note - - return asyncio.run(_go()) - - def test_minor_w8_exactly_120_passes(): - async def _go(): - plane = _W8Plane([120]) - ok, note = await verify_w8(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - # --------------------------------------------------------------------------- - # Minor R3 — all titles required; count alone insufficient - # --------------------------------------------------------------------------- - - return asyncio.run(_go()) - - test_minor_w8_480_minutes_fails() - test_minor_w8_exactly_120_passes() - - def test_minor_r3_behaviours(): - def test_minor_r3_count_without_titles_fails(): - async def _go(): - titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] - run = {"final_text": "There are 2 items due this week.", "calls": []} - ok, note = await verify_r3( - object(), - {"r3_due_titles": titles, "r3_due_count": 2}, - run, - ) - assert ok is False, note - assert "item contract" in note.lower() - - return asyncio.run(_go()) - - def test_minor_r3_exact_item_contract_passes_in_any_order(): - async def _go(): - titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] - run = { - "final_text": ("item: Onboarding email template stale\nitem: Webhook secret rotation docs missing"), - "calls": [], - } - ok, note = await verify_r3( - object(), - {"r3_due_titles": titles, "r3_due_count": 2}, - run, - ) - assert ok is True, note - - return asyncio.run(_go()) - - test_minor_r3_count_without_titles_fails() - test_minor_r3_exact_item_contract_passes_in_any_order() - - test_minor_w8_behaviours() - test_minor_r3_behaviours() +@pytest.mark.parametrize( + ("duration", "want", "expect"), + [ + pytest.param(480, False, "120", id="480-minutes-fails"), + pytest.param(120, True, "", id="exactly-120-passes"), + ], +) +def test_minor_w8_behaviours(duration, want, expect): + ok, note = asyncio.run( + verify_w8( + _W8Plane([duration]), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect: + assert expect in note + + +@pytest.mark.parametrize( + ("run", "want", "expect"), + [ + pytest.param( + {"final_text": "There are 2 items due this week.", "calls": []}, + False, + "item contract", + id="count-without-titles-fails", + ), + pytest.param( + { + "final_text": "item: Onboarding email template stale\nitem: Webhook secret rotation docs missing", + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + "call_source": "test", + "evidence_trace_available": True, + }, + True, + "", + id="exact-items-any-order", + ), + ], +) +def test_minor_r3_behaviours(run, want, expect): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + ok, note = asyncio.run(verify_r3(object(), {"r3_due_titles": titles, "r3_due_count": 2}, run)) + assert ok is want, note + if expect: + assert expect in note.lower() class _S5Plane: @@ -557,30 +673,24 @@ def __init__( ) -def test_s5_requires_all_three_features_not_a_majority(): - """Every two-of-three combination must fail; the task asks for all three.""" - - async def _go(): - cases = [ - ("cycles+customers, worklogs off", True, False, True, False, ("is_time_tracking_enabled",)), - ("worklogs+customers, cycles off", False, True, True, False, ("cycle_view",)), - ("customers only", False, False, True, False, ("cycle_view", "is_time_tracking_enabled")), - ("project flags only, customers off", True, True, False, True, ("customers",)), - ] - for label, cycle_view, tracking, customers, features_cycles, expect in cases: - plane = _S5Plane( - cycle_view=cycle_view, - time_tracking=tracking, - customers=customers, - features_cycles=features_cycles, - ) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is False, f"{label}: {note}" - for s in expect: - assert s in note, f"{label}: {note}" - - plane = _S5Plane(cycle_view=True, time_tracking=True, customers=True, features_cycles=True) - ok, note = await verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) - assert ok is True, note - - return asyncio.run(_go()) +@pytest.mark.parametrize( + ("cycle_view", "tracking", "customers", "features_cycles", "want", "expect"), + [ + pytest.param(True, False, True, False, False, ("is_time_tracking_enabled",), id="worklogs-off"), + pytest.param(False, True, True, False, False, ("cycle_view",), id="cycles-off"), + pytest.param(False, False, True, False, False, ("cycle_view", "is_time_tracking_enabled"), id="customers-only"), + pytest.param(True, True, False, True, False, ("customers",), id="customers-off"), + pytest.param(True, True, True, True, True, (), id="all-three-enabled"), + ], +) +def test_s5_requires_all_three_features_not_a_majority(cycle_view, tracking, customers, features_cycles, want, expect): + plane = _S5Plane( + cycle_view=cycle_view, + time_tracking=tracking, + customers=customers, + features_cycles=features_cycles, + ) + ok, note = asyncio.run(verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + for text in expect: + assert text in note, note From 05debe9fa08adc4341088620c3d902ed275e8c9f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 18:32:42 +0530 Subject: [PATCH 34/93] Report what a run actually measured, not what it attempted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dirty workspace that skipped every task produced "35 skip, 0 pass, 0 fail" and exited 0 — a green battery in which nothing was evaluated. Rows that error or skip correctly leave the success denominator, since an infrastructure fault is not a model failure, but nothing asserted the run had happened. RUN COMPLETE is now reported independently of the success rate and gates the exit status: every expected row reached a terminal outcome, with no infrastructure, harness or cleanup errors and no unexpected skips. EXECUTION COVERAGE is reported separately, so a workspace lacking a capability can finish honestly at less than full coverage. Cleanup failures are aggregated after every target is attempted, because teardown previously failed silently and still reported complete. Skip classification is explicit: a capability the environment does not provide is expected and reduces coverage; a dirty environment or an unrecognised reason is unexpected and breaks completeness. There is no env:* catch-all, and plan-gated capabilities are checked against an allowlist derived from the actual gate sites. Listing activities no longer converts auth failures or timeouts into a missing worker. The canary reports verified, skipped and errored ids rather than claiming "all", fails on teardown error, and tests plausible fabricated answers rather than only an empty one. Statistics: every call statistic is conditioned on success, so the reported minimum can no longer exceed the first quartile; the ad-hoc noise floor is replaced by a paired bootstrap interval and a permutation test that retains ties; tool frequency states its denominator and excluded reps. The author-bias split is removed as confounded — those tasks differ in shape as well as authorship. Co-Authored-By: Claude Opus 5 (1M context) --- evals/errors.py | 14 + evals/report/__init__.py | 24 +- evals/report/__main__.py | 2 +- evals/report/command.py | 26 +- evals/report/compare.py | 67 +- evals/report/load.py | 63 +- evals/report/statistics.py | 102 +- evals/report/summary.py | 184 +++- evals/report/table.py | 169 +-- evals/runner/__init__.py | 2 - evals/runner/canary.py | 139 ++- evals/runner/live.py | 80 +- evals/runner/meta.py | 6 +- evals/runner/resume.py | 11 +- evals/skip_taxonomy.py | 79 ++ tests/evals/report/test_compare.py | 150 +-- tests/evals/report/test_load.py | 181 ++-- tests/evals/report/test_summary.py | 298 +++-- tests/evals/report/test_table.py | 457 ++++---- tests/evals/runner/test_canary.py | 252 +++-- tests/evals/runner/test_live.py | 1611 +++++++++++++++------------- tests/evals/runner/test_resume.py | 457 ++++---- 22 files changed, 2727 insertions(+), 1647 deletions(-) create mode 100644 evals/errors.py create mode 100644 evals/skip_taxonomy.py diff --git a/evals/errors.py b/evals/errors.py new file mode 100644 index 00000000..54520998 --- /dev/null +++ b/evals/errors.py @@ -0,0 +1,14 @@ +"""Neutral evaluation control-flow exceptions.""" + +from __future__ import annotations + + +class TaskSkipped(Exception): + """A task that cannot run in this environment without blaming the agent.""" + + def __init__(self, reason: str) -> None: + self.reason = str(reason) + super().__init__(self.reason) + + +__all__ = ["TaskSkipped"] diff --git a/evals/report/__init__.py b/evals/report/__init__.py index 8a1e7a3e..7f98faaf 100644 --- a/evals/report/__init__.py +++ b/evals/report/__init__.py @@ -9,16 +9,27 @@ is_infra_error_row, is_meta_row, load_rows, + load_run_expected_rows, read_result, ) -from .statistics import iqr, median, percentile, sign_test_pvalue, wilson_interval -from .summary import ResultTokensMode, Summary, TaskSummary, noise_floor_statement, result_tokens_mode, summarize +from .statistics import iqr, median, paired_bootstrap_mean_ci, paired_permutation_pvalue, percentile, wilson_interval +from .summary import ( + ResultTokensMode, + Summary, + TaskSummary, + completeness_statement, + execution_coverage_statement, + result_tokens_mode, + summarize, +) from .table import ( build_multi_surface_table, format_multi_rep_surface_cell, format_number, format_result_tokens, format_surface_cell, + format_tool_distribution, + format_tool_variability, print_table, prompt_excerpt, render_multi_surface_table, @@ -36,18 +47,24 @@ "TaskSummary", "ab_compare", "build_multi_surface_table", + "completeness_statement", "dedupe_rows_latest", "format_multi_rep_surface_cell", "format_number", "format_result_tokens", "format_surface_cell", + "format_tool_distribution", + "format_tool_variability", + "execution_coverage_statement", "iqr", "is_infra_error_row", "is_meta_row", "load_rows", + "load_run_expected_rows", "main", "median", - "noise_floor_statement", + "paired_bootstrap_mean_ci", + "paired_permutation_pvalue", "percentile", "print_ab_report", "print_table", @@ -56,7 +73,6 @@ "render_multi_surface_table", "result_tokens_marker", "result_tokens_mode", - "sign_test_pvalue", "summarize", "surface_label_for_file", "task_sort_key", diff --git a/evals/report/__main__.py b/evals/report/__main__.py index e529f94a..7c4483a6 100644 --- a/evals/report/__main__.py +++ b/evals/report/__main__.py @@ -2,7 +2,7 @@ Usage: python -m evals.report evals/output/A.jsonl - python -m evals.report A.jsonl B.jsonl # A/B delta (sign test + Wilson) + python -m evals.report A.jsonl B.jsonl # paired A/B bootstrap + permutation python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface python -m evals.report --table --markdown f1.jsonl f2.jsonl """ diff --git a/evals/report/command.py b/evals/report/command.py index fb194d1b..9a80f7e3 100644 --- a/evals/report/command.py +++ b/evals/report/command.py @@ -9,7 +9,7 @@ from evals.results import TaskResult from .compare import ab_compare, print_ab_report -from .load import DedupeMode, load_rows +from .load import DedupeMode, load_rows, load_run_expected_rows from .summary import summarize from .table import ( build_multi_surface_table, @@ -60,6 +60,7 @@ def main(argv: list[str] | None = None) -> int: print("error: --table requires at least one JSONL", file=sys.stderr) return 2 labeled: list[tuple[str, list[TaskResult]]] = [] + expected_by_label: dict[str, int | None] = {} used_labels: set[str] = set() for path in paths: rows = load_rows(path, dedupe=dedupe) @@ -72,29 +73,30 @@ def main(argv: list[str] | None = None) -> int: number += 1 used_labels.add(label) labeled.append((label, rows)) + expected_by_label[label] = load_run_expected_rows(path) warn_if_table_mixes_batteries(labeled) - table = build_multi_surface_table(labeled) + table = build_multi_surface_table(labeled, expected_rows_by_column=expected_by_label) sys.stdout.write(render_multi_surface_table(table, markdown=arguments.markdown)) - return 0 + return 0 if all(values["complete"] for values in table["footer"].values()) else 1 if len(paths) == 1: path = paths[0] rows = load_rows(path, dedupe=dedupe) - summary = summarize(rows) - if not summary.tasks: - if summary.infra_errors: - print(f"infra errors: {summary.infra_errors}") - print(f"(no non-skipped / non-error rows in {path})") - return 0 + summary = summarize(rows, expected_rows=load_run_expected_rows(path)) print_table(summary, f"Summary: {path}") - return 0 + return 0 if summary.complete else 1 if len(paths) == 2: rows_a = load_rows(paths[0], dedupe=dedupe) rows_b = load_rows(paths[1], dedupe=dedupe) - comparison = ab_compare(rows_a, rows_b) + comparison = ab_compare( + rows_a, + rows_b, + expected_rows_a=load_run_expected_rows(paths[0]), + expected_rows_b=load_run_expected_rows(paths[1]), + ) print_ab_report(comparison, paths[0], paths[1]) - return 0 + return 0 if comparison["summary_a"].complete and comparison["summary_b"].complete else 1 print( "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", diff --git a/evals/report/compare.py b/evals/report/compare.py index b1a06ca4..a5a659ab 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -7,23 +7,30 @@ from typing import Any from .load import ResultRow, is_infra_error_row, is_meta_row, read_result -from .statistics import median, sign_test_pvalue -from .summary import noise_floor_statement, summarize +from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue +from .summary import completeness_statement, execution_coverage_statement, summarize from .table import format_number def ab_compare( rows_a: list[ResultRow], rows_b: list[ResultRow], + *, + expected_rows_a: int | None = None, + expected_rows_b: int | None = None, ) -> dict[str, Any]: - """Compare two result sets: paired call-count deltas + success rates. + """Compare two result sets with task-paired call and success deltas. Paired call deltas only include tasks with at least one successful repetition in both A and B. Calls are the median across successful repetitions; this is identical to the historical behavior for single-rep files. + + Success-rate differences pair each task's completed-repetition rate across + labels, then bootstrap whole task pairs. This treats tasks as independent + sampling units and assumes the labels cover comparable task instances. """ - summary_a = summarize(rows_a) - summary_b = summarize(rows_b) + summary_a = summarize(rows_a, expected_rows=expected_rows_a) + summary_b = summarize(rows_b, expected_rows=expected_rows_b) def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: output: dict[str, list[float]] = defaultdict(list) @@ -55,16 +62,33 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: } ) + success_shared = sorted( + task_id + for task_id in set(summary_a.tasks) & set(summary_b.tasks) + if summary_a.tasks[task_id].n and summary_b.tasks[task_id].n + ) + success_deltas = [ + (summary_b.tasks[task_id].k / summary_b.tasks[task_id].n) + - (summary_a.tasks[task_id].k / summary_a.tasks[task_id].n) + for task_id in success_shared + ] + paired_success_delta = sum(success_deltas) / len(success_deltas) if success_deltas else None + paired_success_ci = paired_bootstrap_mean_ci(success_deltas) + return { "summary_a": summary_a, "summary_b": summary_b, "paired_tasks": per_task, + "mean_delta": sum(deltas) / len(deltas) if deltas else None, "median_delta": median(deltas), - "sign_test_p": sign_test_pvalue(deltas), + "call_permutation_p": paired_permutation_pvalue(deltas), + "call_zero_deltas": sum(delta == 0 for delta in deltas), "n_paired": len(deltas), + "paired_success_tasks": success_shared, + "n_paired_success": len(success_deltas), + "paired_success_delta": paired_success_delta, + "paired_success_ci": paired_success_ci, "multi_rep": summary_a.multi_rep or summary_b.multi_rep, - "unstable_a": summary_a.unstable_tasks, - "unstable_b": summary_b.unstable_tasks, "success_a": { "k": summary_a.aggregate_k, "n": summary_a.aggregate_n, @@ -97,15 +121,32 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N f" success B: {success_b['k']}/{success_b['n']} ({rate_b:.1%}) " f"Wilson95 [{success_b['wilson'][0]:.2f},{success_b['wilson'][1]:.2f}]" ) + print(f" A {execution_coverage_statement(comparison['summary_a'])}") + print(f" B {execution_coverage_statement(comparison['summary_b'])}") + print(f" A {completeness_statement(comparison['summary_a'])}") + print(f" B {completeness_statement(comparison['summary_b'])}") print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") + paired_success_delta = comparison["paired_success_delta"] + paired_success_lo, paired_success_hi = comparison["paired_success_ci"] + if paired_success_delta is None or paired_success_lo is None or paired_success_hi is None: + print(" paired task success delta (B−A): n/a (no shared evaluated tasks)") + else: + print( + f" paired task success delta (B−A): {paired_success_delta:+.1%} " + f"paired-bootstrap95 [{paired_success_lo:+.1%},{paired_success_hi:+.1%}] " + f"(n={comparison['n_paired_success']} tasks)" + ) print(f" paired successful tasks: {comparison['n_paired']}") + print(f" mean call delta (B−A): {format_number(comparison['mean_delta'])}") print(f" median call delta (B−A): {format_number(comparison['median_delta'])}") - probability = comparison["sign_test_p"] - print(f" sign-test p-value (two-sided): {probability if probability is not None else 'n/a'}") + probability = comparison["call_permutation_p"] + tie_count = comparison["call_zero_deltas"] + print( + " paired permutation p-value for mean call delta (two-sided): " + f"{probability if probability is not None else 'n/a'} " + f"({tie_count} zero-delta ties retained)" + ) multiple_repetitions = bool(comparison.get("multi_rep")) - if multiple_repetitions: - print(f" A {noise_floor_statement(int(comparison.get('unstable_a') or 0))}") - print(f" B {noise_floor_statement(int(comparison.get('unstable_b') or 0))}") if comparison["paired_tasks"]: print() print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") diff --git a/evals/report/load.py b/evals/report/load.py index fbefa6ea..3742ccc0 100644 --- a/evals/report/load.py +++ b/evals/report/load.py @@ -1,4 +1,4 @@ -"""JSONL row loading and classification for evaluation reports.""" +"""JSONL row loading and error handling for evaluation reports.""" from __future__ import annotations @@ -13,6 +13,38 @@ ResultRow = TaskResult | dict[str, Any] +def _invalid_result_row(path: Path, line_number: int, reason: str) -> TaskResult: + """Represent an unreadable persisted row as a completeness-visible harness error.""" + return TaskResult( + task_id=f"", + rep=line_number, + label=path.stem, + success=False, + error=f"{path}:{line_number}: {reason}", + error_class="harness_report_load", + ) + + +def load_run_expected_rows(path: Path) -> int | None: + """Read the declared run size from the JSONL meta header, when available.""" + with path.open(encoding="utf-8") as file: + for line in file: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(row, dict): + continue + if row.get("row_type") != "meta": + return None + value = row.get("expected_rows") + return int(value) if value is not None else None + return None + + def read_result(row: ResultRow) -> TaskResult: """Return one row as the declared persisted result type.""" return row if isinstance(row, TaskResult) else TaskResult.from_row(row) @@ -48,7 +80,7 @@ def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: - """Load JSONL data rows (skip meta / missing task_id). + """Load JSONL data rows, representing malformed data as harness errors. Default ``dedupe="latest"`` keeps the last row per (task_id, rep, label) so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. @@ -63,14 +95,35 @@ def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: row = json.loads(line) except json.JSONDecodeError as exc: print( - f"warning: {path}:{line_number}: skipping invalid JSON ({exc})", + f"warning: {path}:{line_number}: recording invalid JSON as a harness error ({exc})", file=sys.stderr, ) + rows.append(_invalid_result_row(path, line_number, f"invalid JSON ({exc})")) continue if not isinstance(row, dict): + print( + f"warning: {path}:{line_number}: recording non-object JSON as a harness error", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, "result row is not a JSON object")) + continue + if row.get("row_type") == "meta": + continue + try: + result = TaskResult.from_row(row) + except Exception as exc: + print( + f"warning: {path}:{line_number}: recording invalid result object as a harness error ({exc})", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, f"invalid result object ({exc})")) continue - result = TaskResult.from_row(row) - if is_meta_row(result): + if not result.task_id: + print( + f"warning: {path}:{line_number}: recording result without task_id as a harness error", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, "result row has no task_id")) continue rows.append(result) if dedupe == "latest": diff --git a/evals/report/statistics.py b/evals/report/statistics.py index 33ee0546..a91239af 100644 --- a/evals/report/statistics.py +++ b/evals/report/statistics.py @@ -2,7 +2,13 @@ from __future__ import annotations +import itertools import math +import random + +EXACT_PERMUTATION_LIMIT = 20 +MONTE_CARLO_PERMUTATIONS = 100_000 +BOOTSTRAP_RESAMPLES = 20_000 def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: @@ -19,22 +25,92 @@ def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: return (lo, hi) -def sign_test_pvalue(deltas: list[float]) -> float | None: - """Two-sided exact binomial sign test on non-zero paired deltas. +def paired_permutation_pvalue( + deltas: list[float], + *, + exact_limit: int = EXACT_PERMUTATION_LIMIT, + permutations: int = MONTE_CARLO_PERMUTATIONS, + seed: int = 0, +) -> float | None: + """Two-sided paired sign-flip permutation test on the mean delta. + + The null assumes each pair's A/B labels are exchangeable and pairs are + independent. The statistic is the absolute mean of *all* paired deltas, so + zero-delta ties remain in the sample and its denominator. A zero contributes + the same value under either sign; enumerating its duplicate sign assignments + once is exactly equivalent to enumerating both. - H0: P(delta > 0) = 1/2. Zero deltas are dropped. Returns None when no - non-zero pairs remain. Uses ``math.comb`` only (no scipy). + Tests with at most ``exact_limit`` non-zero contributions enumerate the exact + randomization distribution. Larger tests use a deterministic Monte Carlo + sample and the standard plus-one correction. ``None`` means there were no + pairs; an all-tie sample returns 1.0. """ - nonzero = [d for d in deltas if d != 0] - n = len(nonzero) - if n == 0: + if not deltas: return None - k = sum(1 for d in nonzero if d > 0) - total = 2**n - # Two-sided: 2 * min(left cdf, right survival), capped at 1. - left = sum(math.comb(n, i) for i in range(0, k + 1)) / total - right = sum(math.comb(n, i) for i in range(k, n + 1)) / total - return min(1.0, 2.0 * min(left, right)) + pair_count = len(deltas) + contributions = [float(delta) for delta in deltas if delta != 0] + observed = abs(sum(deltas) / pair_count) + tolerance = 1e-12 + if not contributions: + return 1.0 + + def is_extreme(signs: tuple[int, ...] | list[int]) -> bool: + permuted = abs(sum(sign * delta for sign, delta in zip(signs, contributions, strict=True)) / pair_count) + return permuted + tolerance >= observed + + if len(contributions) <= exact_limit: + assignments = itertools.product((-1, 1), repeat=len(contributions)) + extreme = sum(1 for signs in assignments if is_extreme(signs)) + return extreme / (2 ** len(contributions)) + + if permutations <= 0: + raise ValueError("permutations must be positive") + generator = random.Random(seed) + extreme = 0 + for _ in range(permutations): + signs = [generator.choice((-1, 1)) for _ in contributions] + extreme += is_extreme(signs) + return (extreme + 1) / (permutations + 1) + + +def paired_bootstrap_mean_ci( + deltas: list[float], + *, + confidence: float = 0.95, + resamples: int = BOOTSTRAP_RESAMPLES, + seed: int = 0, +) -> tuple[float | None, float | None]: + """Percentile paired-bootstrap CI for the mean per-pair delta. + + Resampling whole paired deltas preserves the A/B pairing. The interval treats + tasks as independent sampling units drawn from a task population and assumes + the two labels measured comparable task instances. It captures task-sampling + uncertainty, not dependence between tasks or systematic run/environment drift. + + Small samples are intentionally not narrowed by row-level repetitions: for up + to five pairs the complete ``n**n`` bootstrap distribution is enumerated. + Larger samples use a deterministic Monte Carlo bootstrap. + """ + if not deltas: + return (None, None) + if not 0.0 < confidence < 1.0: + raise ValueError("confidence must be between 0 and 1") + if resamples <= 0: + raise ValueError("resamples must be positive") + + values = [float(delta) for delta in deltas] + sample_size = len(values) + bootstrap_means: list[float] = [] + if sample_size**sample_size <= resamples: + for sample in itertools.product(values, repeat=sample_size): + bootstrap_means.append(sum(sample) / sample_size) + else: + generator = random.Random(seed) + for _ in range(resamples): + bootstrap_means.append(sum(generator.choice(values) for _ in range(sample_size)) / sample_size) + + tail = (1.0 - confidence) / 2.0 + return (percentile(bootstrap_means, tail), percentile(bootstrap_means, 1.0 - tail)) def median(values: list[float]) -> float | None: diff --git a/evals/report/summary.py b/evals/report/summary.py index fd8405de..14c9888e 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -7,7 +7,7 @@ from typing import Literal from evals.results import TaskResult -from evals.tasks import TASKS_BY_ID +from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import iqr, median, percentile, wilson_interval @@ -27,8 +27,10 @@ class TaskSummary: calls_max: float | None calls_q1: float | None calls_q3: float | None - optimal_calls: int | None - mispick_rate: float + tool_reps: int + failed_tool_reps: int + tool_rep_frequency: dict[str, float] + tool_call_counts: dict[str, int] errored_calls: int capped: int harness_err: int @@ -47,11 +49,29 @@ def unstable(self) -> bool: def success(self) -> str: return f"{self.k}/{self.n}" if self.n else "0/0" + @property + def tool_distribution_available(self) -> bool: + return self.tool_reps >= 2 + + @property + def variable_tool_names(self) -> list[str]: + return [tool for tool, frequency in self.tool_rep_frequency.items() if frequency < 1.0] + @dataclass(slots=True) class Summary: tasks: dict[str, TaskSummary] + total_tasks: int + expected_rows: int + completed_rows: int infra_errors: int + harness_errors: int + expected_skips: int + unexpected_skips: int + cleanup_errors: int + expected_skip_reasons: dict[str, int] + unexpected_skip_reasons: dict[str, int] + skipped_task_reasons: dict[str, list[str]] aggregate_k: int aggregate_n: int aggregate_wilson_lo: float @@ -59,6 +79,16 @@ class Summary: multi_rep: bool result_tokens_mode: ResultTokensMode + @property + def complete(self) -> bool: + return ( + self.completed_rows == self.expected_rows + and self.infra_errors == 0 + and self.harness_errors == 0 + and self.unexpected_skips == 0 + and self.cleanup_errors == 0 + ) + @property def unstable_task_ids(self) -> list[str]: return [task_id for task_id, task in self.tasks.items() if task.unstable] @@ -67,6 +97,14 @@ def unstable_task_ids(self) -> list[str]: def unstable_tasks(self) -> int: return len(self.unstable_task_ids) + @property + def variable_tool_tasks(self) -> int: + return sum(bool(task.variable_tool_names) for task in self.tasks.values()) + + @property + def tool_distribution_available(self) -> bool: + return any(task.tool_distribution_available for task in self.tasks.values()) + def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: """Classify token counts without treating unmarked legacy data as measured.""" @@ -98,7 +136,46 @@ def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: return "mixed" -def summarize(rows: list[ResultRow]) -> Summary: +def _format_reason_counts(reasons: dict[str, int]) -> str: + return ", ".join(f"{reason}={count}" for reason, count in sorted(reasons.items())) + + +def completeness_statement(summary: Summary) -> str: + """Render completeness independently from the model success rate.""" + prefix = "RUN COMPLETE" if summary.complete else "RUN INCOMPLETE" + parts = [f"{summary.completed_rows}/{summary.expected_rows} rows completed"] + if summary.infra_errors: + parts.append(f"infra errors={summary.infra_errors}") + if summary.harness_errors: + parts.append(f"harness errors={summary.harness_errors}") + if summary.unexpected_skips: + reasons = _format_reason_counts(summary.unexpected_skip_reasons) + parts.append(f"unexpected skips={summary.unexpected_skips} [{reasons}]") + if summary.cleanup_errors: + parts.append(f"cleanup errors={summary.cleanup_errors}") + if summary.expected_skips: + reasons = _format_reason_counts(summary.expected_skip_reasons) + parts.append(f"expected skips={summary.expected_skips} [{reasons}]") + return f"{prefix}: " + "; ".join(parts) + + +def execution_coverage_statement(summary: Summary) -> str: + """Render rows actually evaluated, independently from run completeness.""" + if summary.expected_rows: + rate = summary.aggregate_n / summary.expected_rows + amount = f"{summary.aggregate_n}/{summary.expected_rows} rows evaluated ({rate:.1%})" + else: + amount = f"{summary.aggregate_n}/0 rows evaluated (n/a)" + parts = [amount] + if summary.skipped_task_reasons: + skips = "; ".join( + f"{','.join(task_ids)} ({reason})" for reason, task_ids in sorted(summary.skipped_task_reasons.items()) + ) + parts.append(f"skipped tasks=[{skips}]") + return "EXECUTION COVERAGE: " + "; ".join(parts) + + +def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Summary: """Aggregate per-task metrics. Rows with ``error_class`` starting ``infra_`` are excluded from success-rate @@ -111,21 +188,44 @@ def summarize(rows: list[ResultRow]) -> Summary: infrastructure_errors_by_task: dict[str, int] = defaultdict(int) repetitions_by_task: dict[str, set[int]] = defaultdict(set) infrastructure_errors = 0 + harness_errors = 0 + completed_rows = 0 + expected_skips = 0 + unexpected_skips = 0 + cleanup_errors = 0 + expected_skip_reasons: dict[str, int] = defaultdict(int) + unexpected_skip_reasons: dict[str, int] = defaultdict(int) + skipped_task_reasons: dict[str, set[str]] = defaultdict(set) + declared_expected_rows = 0 for raw_row in rows: row = read_result(raw_row) if is_meta_row(row): continue + declared_expected_rows = max(declared_expected_rows, row.expected_rows) task_id = row.task_id repetitions_by_task[task_id].add(row.rep) + if row.cleanup_error: + cleanup_errors += 1 if is_infra_error_row(row): infrastructure_errors += 1 infrastructure_errors_by_task[task_id] += 1 continue # infra seed/cli — excluded from success aggregates if row.error: + harness_errors += 1 harness_errors_by_task[task_id] += 1 continue # harness/API errors excluded from success/medians (F4) if row.skipped: + family = skip_reason_family(row.skipped) + skipped_task_reasons[row.skipped].add(task_id) + if is_expected_environment_capability_skip(row.skipped): + expected_skips += 1 + expected_skip_reasons[family] += 1 + completed_rows += 1 + else: + unexpected_skips += 1 + unexpected_skip_reasons[family] += 1 continue # skipped rows are excluded from success denominators + completed_rows += 1 by_task[task_id].append(row) # Include tasks that only had harness/infra errors so columns stay visible. @@ -141,20 +241,36 @@ def summarize(rows: list[ResultRow]) -> Summary: total_passes += pass_count total_repetitions += repetition_count lower, upper = wilson_interval(pass_count, repetition_count) if repetition_count else (0.0, 0.0) - calls = [float(row.num_calls) for row in task_results] - first_quartile, median_calls, third_quartile = iqr(calls) - minimum_calls = min(calls) if calls else None - maximum_calls = max(calls) if calls else None - optimal = TASKS_BY_ID.get(task_id, {}).get("optimal_calls") - total_calls = 0 - mispicks = 0 + successful_calls = [float(row.num_calls) for row in task_results if row.success] + first_quartile, median_calls, third_quartile = iqr(successful_calls) + minimum_calls = min(successful_calls) if successful_calls else None + maximum_calls = max(successful_calls) if successful_calls else None + successful_results = [row for row in task_results if row.success] + tool_reps = len(successful_results) + failed_tool_reps = ( + repetition_count + - tool_reps + + harness_errors_by_task.get(task_id, 0) + + infrastructure_errors_by_task.get(task_id, 0) + ) + tool_rep_counts: dict[str, int] = defaultdict(int) + tool_call_counts: dict[str, int] = defaultdict(int) + for row in successful_results: + tools_in_rep: set[str] = set() + for call in row.calls: + if not call.tool: + continue + tool_call_counts[call.tool] += 1 + tools_in_rep.add(call.tool) + for tool in tools_in_rep: + tool_rep_counts[tool] += 1 + tool_rep_frequency = ( + {tool: tool_rep_counts[tool] / tool_reps for tool in sorted(tool_rep_counts)} if tool_reps >= 2 else {} + ) errored_calls = 0 result_tokens: list[float] = [] for row in task_results: for call in row.calls: - total_calls += 1 - if call.classification in ("alternate", "out_of_set"): - mispicks += 1 if call.is_error: errored_calls += 1 if call.result_tokens is not None: @@ -172,8 +288,10 @@ def summarize(rows: list[ResultRow]) -> Summary: calls_max=maximum_calls, calls_q1=first_quartile, calls_q3=third_quartile, - optimal_calls=optimal, - mispick_rate=(mispicks / total_calls) if total_calls else 0.0, + tool_reps=tool_reps, + failed_tool_reps=failed_tool_reps, + tool_rep_frequency=tool_rep_frequency, + tool_call_counts={tool: tool_call_counts[tool] for tool in sorted(tool_call_counts)}, errored_calls=errored_calls, capped=capped, harness_err=harness_errors_by_task.get(task_id, 0), @@ -186,9 +304,24 @@ def summarize(rows: list[ResultRow]) -> Summary: aggregate_lower, aggregate_upper = ( wilson_interval(total_passes, total_repetitions) if total_repetitions else (0.0, 0.0) ) + resolved_expected_rows = ( + max(expected_rows, declared_expected_rows) + if expected_rows is not None + else declared_expected_rows or sum(1 for row in rows if not is_meta_row(row)) + ) return Summary( tasks=output, + total_tasks=len(repetitions_by_task), + expected_rows=resolved_expected_rows, + completed_rows=completed_rows, infra_errors=infrastructure_errors, + harness_errors=harness_errors, + expected_skips=expected_skips, + unexpected_skips=unexpected_skips, + cleanup_errors=cleanup_errors, + expected_skip_reasons=dict(expected_skip_reasons), + unexpected_skip_reasons=dict(unexpected_skip_reasons), + skipped_task_reasons={reason: sorted(task_ids) for reason, task_ids in sorted(skipped_task_reasons.items())}, aggregate_k=total_passes, aggregate_n=total_repetitions, aggregate_wilson_lo=aggregate_lower, @@ -196,22 +329,3 @@ def summarize(rows: list[ResultRow]) -> Summary: multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), ) - - -def noise_floor_statement(unstable_tasks: int) -> str: - """Describe observed pass/fail variance in task-count comparison units.""" - count = max(0, int(unstable_tasks)) - if count == 0: - return ( - "measured noise floor: 0 tasks flipped at least once; no non-zero " - "run-to-run variance was observed (minimum meaningful difference " - "from observed flips: 1 task)" - ) - noun = "task" if count == 1 else "tasks" - threshold = count + 1 - threshold_noun = "task" if threshold == 1 else "tasks" - return ( - f"measured noise floor: {count} {noun} flipped at least once; surface " - f"differences of {count} {noun} or fewer are within observed run-to-run " - f"variance (minimum meaningful difference: {threshold} {threshold_noun})" - ) diff --git a/evals/report/table.py b/evals/report/table.py index d327d9af..9b602bb4 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -12,7 +12,13 @@ from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import wilson_interval -from .summary import Summary, noise_floor_statement +from .summary import ( + Summary, + TaskSummary, + completeness_statement, + execution_coverage_statement, + summarize, +) def format_number(value: float | None, digits: int = 1) -> str: @@ -32,6 +38,39 @@ def format_result_tokens(value: float | None, mode: str) -> str: return f"{result_tokens_marker(mode)}{formatted}" +def format_tool_distribution(task: TaskSummary) -> str: + """Render success-conditioned tool frequency with its exclusions visible.""" + conditioning = f"success-only n={task.tool_reps}; failed excluded={task.failed_tool_reps}" + if not task.tool_distribution_available: + return f"{conditioning}; frequency=—" + if not task.tool_rep_frequency: + return f"{conditioning}; no tools" + core = [ + f"{tool}({task.tool_call_counts[tool]}c)" + for tool, frequency in task.tool_rep_frequency.items() + if frequency == 1.0 + ] + variable = [ + f"{tool}={frequency:.0%}({task.tool_call_counts[tool]}c)" + for tool, frequency in task.tool_rep_frequency.items() + if frequency < 1.0 + ] + groups = [] + if core: + groups.append(f"core:{','.join(core)}") + if variable: + groups.append(f"variable:{','.join(variable)}") + return f"{conditioning}; {'; '.join(groups)}" + + +def format_tool_variability(summary: Summary, total_tasks: int | None = None) -> str: + """Render the fleet count of tasks with variable tool use.""" + if not summary.tool_distribution_available: + return "—" + total = summary.total_tasks if total_tasks is None else total_tasks + return f"{summary.variable_tool_tasks}/{total} tasks" + + def print_table(summary: Summary, title: str) -> None: print(title) token_mode = summary.result_tokens_mode @@ -41,8 +80,6 @@ def print_table(summary: Summary, title: str) -> None: print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") elif token_mode == "unlabeled": print("result-token columns marked ?: include legacy values with unknown measurement status") - if summary.infra_errors: - print(f"infra errors: {summary.infra_errors}") aggregate_count = summary.aggregate_n if aggregate_count: aggregate_passes = summary.aggregate_k @@ -52,9 +89,14 @@ def print_table(summary: Summary, title: str) -> None: print( f"aggregate success: {aggregate_passes}/{aggregate_count} ({rate:.1%}) Wilson95 [{lower:.2f},{upper:.2f}]" ) + else: + print("aggregate success: 0/0 (n/a; no evaluated rows)") + print(execution_coverage_statement(summary)) + print(completeness_statement(summary)) + if summary.infra_errors: + print(f"infra errors: {summary.infra_errors}") + print(f"tool variability: {format_tool_variability(summary)}") multiple_repetitions = summary.multi_rep - if multiple_repetitions: - print(noise_floor_statement(summary.unstable_tasks)) # Multi-rep files keep the repetition-aware layout even when errors leave # only one completed result in every task's success-rate denominator. show_variation = multiple_repetitions or any(task.n > 1 for task in summary.tasks.values()) @@ -66,26 +108,26 @@ def print_table(summary: Summary, title: str) -> None: header = ( f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " f"{unstable_header}" - f"{'calls_min':>9} {'med_calls':>9} {'calls_max':>9} {'opt':>4} " - f"{'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'success_calls_min':>17} {'success_calls_med':>17} " + f"{'success_calls_max':>17} {'success_calls_q1-q3':>19} {'err':>4} " f"{'capped':>6} {'h_err':>5} {'i_err':>5} " f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " - f"{'med_cum_in':>10}" + f"{'med_cum_in':>10} tool distribution" ) else: header = ( f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " - f"{'med_calls':>9} {'opt':>4} {'IQR':>11} {'mispick':>8} {'err':>4} " + f"{'success_calls_med':>17} {'success_calls_min':>17} " + f"{'success_calls_q1-q3':>19} {'err':>4} " f"{'capped':>6} {'h_err':>5} {'i_err':>5} " f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " - f"{'med_cum_in':>10}" + f"{'med_cum_in':>10} tool distribution" ) print(header) print("-" * len(header)) for task_id, values in summary.tasks.items(): wilson = f"[{values.wilson_lo:.2f},{values.wilson_hi:.2f}]" quartiles = f"{format_number(values.calls_q1)}-{format_number(values.calls_q3)}" - optimal = values.optimal_calls if values.optimal_calls is not None else "-" task_token_mode = values.result_tokens_mode if show_variation: unstable = ("YES" if values.unstable else "no") if multiple_repetitions else "" @@ -93,26 +135,24 @@ def print_table(summary: Summary, title: str) -> None: print( f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " f"{unstable_cell}" - f"{format_number(values.calls_min):>9} " - f"{format_number(values.med_calls):>9} " - f"{format_number(values.calls_max):>9} " - f"{optimal!s:>4} {quartiles:>11} {values.mispick_rate:>7.1%} " - f"{values.errored_calls:>4} {values.capped:>6} " + f"{format_number(values.calls_min):>17} " + f"{format_number(values.med_calls):>17} " + f"{format_number(values.calls_max):>17} " + f"{quartiles:>19} {values.errored_calls:>4} {values.capped:>6} " f"{values.harness_err:>5} {values.infra_err:>5} " f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " - f"{format_number(values.med_cum_input, 0):>10}" + f"{format_number(values.med_cum_input, 0):>10} {format_tool_distribution(values)}" ) else: print( f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " - f"{format_number(values.med_calls):>9} {optimal!s:>4} " - f"{quartiles:>11} {values.mispick_rate:>7.1%} " - f"{values.errored_calls:>4} {values.capped:>6} " + f"{format_number(values.med_calls):>17} {format_number(values.calls_min):>17} " + f"{quartiles:>19} {values.errored_calls:>4} {values.capped:>6} " f"{values.harness_err:>5} {values.infra_err:>5} " f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " - f"{format_number(values.med_cum_input, 0):>10}" + f"{format_number(values.med_cum_input, 0):>10} {format_tool_distribution(values)}" ) @@ -122,7 +162,7 @@ def task_sort_key(task_id: str) -> tuple[str, int]: def format_surface_cell(row: ResultRow | None) -> str: - """Cell for multi-surface table: '✅ Nc/Mmp', 'skip', 'ERR', or '—'.""" + """Cell for a single-repetition surface result.""" if row is None: return "—" result = read_result(row) @@ -132,17 +172,7 @@ def format_surface_cell(row: ResultRow | None) -> str: return "ERR" passed = "✅" if result.success else "❌" call_count = str(result.num_calls) - if result.server == "external": - return f"{passed} {call_count}c" - alternate = result.alternate_calls - outside_set = result.out_of_set_calls - # None counters (external nulling) → omit mispick suffix. - if alternate is None and outside_set is None: - return f"{passed} {call_count}c" - mispicks = int(alternate or 0) + int(outside_set or 0) - if mispicks: - return f"{passed} {call_count}c/{mispicks}mp" - return f"{passed} {call_count}c" + return f"{passed} {call_count}c · tools —" def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: @@ -161,7 +191,9 @@ def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: marker = "❌" calls = [row.num_calls for row in completed] call_span = f"{min(calls)}c" if min(calls) == max(calls) else f"{min(calls)}-{max(calls)}c" - return f"{marker} {pass_count}/{repetition_count} [{lower:.2f},{upper:.2f}] {call_span}" + task_summary = summarize(results).tasks[results[0].task_id] + tools = format_tool_distribution(task_summary) + return f"{marker} {pass_count}/{repetition_count} [{lower:.2f},{upper:.2f}] {call_span} · tools {tools}" if any(row.error or is_infra_error_row(row) for row in results): return "ERR" if any(row.skipped for row in results): @@ -171,6 +203,8 @@ def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: def build_multi_surface_table( file_rows: list[tuple[str, list[ResultRow]]], + *, + expected_rows_by_column: dict[str, int | None] | None = None, ) -> dict[str, Any]: """Build a per-task × per-surface grid from labeled row sets. @@ -217,12 +251,9 @@ def build_multi_surface_table( # Aggregate footer per column. footer: dict[str, dict[str, Any]] = {} for column in columns: - successes = repetitions = calls = mispicks = 0 - mispicks_comparable = True + successes = repetitions = calls = 0 infrastructure_errors = 0 - unstable_tasks = 0 for task_rows in rows_by_column[column].values(): - completed: list[TaskResult] = [] for row in task_rows: if is_infra_error_row(row): infrastructure_errors += 1 @@ -231,31 +262,27 @@ def build_multi_surface_table( continue if row.skipped: continue - completed.append(row) repetitions += 1 if row.success: successes += 1 calls += row.num_calls - if row.server == "external": - mispicks_comparable = False - else: - alternate = row.alternate_calls - outside_set = row.out_of_set_calls - if alternate is None and outside_set is None: - mispicks_comparable = False - else: - mispicks += int(alternate or 0) + int(outside_set or 0) - task_passes = sum(1 for row in completed if row.success) - if len(completed) > 1 and 0 < task_passes < len(completed): - unstable_tasks += 1 + column_summary = summarize( + [row for task_rows in rows_by_column[column].values() for row in task_rows], + expected_rows=(expected_rows_by_column or {}).get(column), + ) footer[column] = { "success": successes, "n": repetitions, "calls": calls, - "mispicks": mispicks if mispicks_comparable else None, "infra_errors": infrastructure_errors, "multi_rep": multiple_repetitions_by_column[column], - "unstable_tasks": unstable_tasks, + "tool_variability": ( + column_summary.variable_tool_tasks if column_summary.tool_distribution_available else None + ), + "tasks": len(rows_by_column[column]), + "complete": column_summary.complete, + "completeness": completeness_statement(column_summary), + "coverage": execution_coverage_statement(column_summary), } return { "columns": columns, @@ -295,17 +322,19 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) for column in columns: values = footer[column] rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" - mispicks = f", {values['mispicks']}mp" if values["mispicks"] is not None else "" - footer_parts.append(f"{rate} ({values['calls']}c{mispicks}, i={values['infra_errors']})") + variability = ( + f"{values['tool_variability']}/{values['tasks']} variable" + if values["tool_variability"] is not None + else "tools —" + ) + footer_parts.append(f"{rate} ({values['calls']}c, {variability}, i={values['infra_errors']})") lines.append("| **agg** | | " + " | ".join(footer_parts) + " |") - if multiple_repetitions: - noise_parts = [ - noise_floor_statement(int(footer[column].get("unstable_tasks") or 0)) - if footer[column].get("multi_rep") - else "single repetition" - for column in columns - ] - lines.append("| **noise floor** | | " + " | ".join(noise_parts) + " |") + lines.append( + "| **execution coverage** | | " + " | ".join(footer[column]["coverage"] for column in columns) + " |" + ) + lines.append( + "| **completeness** | | " + " | ".join(footer[column]["completeness"] for column in columns) + " |" + ) return "\n".join(lines) + "\n" column_width = max(14, max((len(column) for column in columns), default=14)) @@ -330,15 +359,17 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) values = footer[column] rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" percentage = f" ({100 * values['success'] / values['n']:.0f}%)" if values["n"] else "" - mispicks = f" mispicks {values['mispicks']}" if values["mispicks"] is not None else " mispicks n/a" + variability = ( + f"{values['tool_variability']}/{values['tasks']} tasks" if values["tool_variability"] is not None else "—" + ) lines.append( f"{column:12} success {rate}{percentage} total calls {values['calls']}" - f"{mispicks} infra {values['infra_errors']}" + f" tool variability {variability} infra {values['infra_errors']}" ) - if multiple_repetitions: - for column in columns: - if footer[column].get("multi_rep"): - lines.append(f"{column:12} {noise_floor_statement(int(footer[column].get('unstable_tasks') or 0))}") + for column in columns: + lines.append(f"{column:12} {footer[column]['coverage']}") + for column in columns: + lines.append(f"{column:12} {footer[column]['completeness']}") return "\n".join(lines) + "\n" diff --git a/evals/runner/__init__.py b/evals/runner/__init__.py index f33008aa..c668940c 100644 --- a/evals/runner/__init__.py +++ b/evals/runner/__init__.py @@ -4,7 +4,6 @@ from .live import ( MAX_ITERATIONS, MAX_TOKENS, - classify_call, is_infra_cli_stop_reason, run_agent_task_via_driver, run_live, @@ -16,7 +15,6 @@ __all__ = [ "MAX_ITERATIONS", "MAX_TOKENS", - "classify_call", "is_infra_cli_stop_reason", "is_meta_or_non_task_row", "load_resume_skip_keys", diff --git a/evals/runner/canary.py b/evals/runner/canary.py index 5672bdcb..dd625bf6 100644 --- a/evals/runner/canary.py +++ b/evals/runner/canary.py @@ -10,69 +10,148 @@ from evals.tasks.catalog import battery_fingerprint from evals.tasks.skip import TaskSkipped +CANARY_CANNED_OUTPUTS: dict[str, tuple[str, ...]] = { + "R1": ("state: In Progress",), + "R2": ("count: 0",), + "R3": ("item: Example work item",), + "R4": ("cycle: Sprint 13\nitem: Example work item\noverdue: none",), + "R5": ("comment: looks good",), + "R6": ("project: EVAL deadbeef B",), + "R7": ("state: Backlog | group: backlog",), + "C2": ("release: 1.2.0\nshipped: guessed change",), + "I2": ("state: Backlog",), + "L1": ("logged-minutes: 90\nsummary-work-item-id: guessed-id",), + "L2": ("count: 0",), + "L5": ("count: 0",), +} + + +def canary_probe_texts(task_id: str) -> tuple[str, ...]: + """Return empty plus plausible zero-call answers for a verifier canary.""" + values = ("", "count: 0", *CANARY_CANNED_OUTPUTS.get(task_id, ())) + return tuple(dict.fromkeys(values)) + async def run_canary( tasks: list[dict[str, Any]], *, label: str, + required_task_ids: set[str] | frozenset[str] | None = None, ) -> int: - """Seed + verify(empty agent) + teardown per task; no driver/model. + """Seed + verify zero-call probes + teardown per task; no driver/model. - Passes only when every verifier returns falsy ok on a do-nothing agent. - Any ok=True is a broken verifier (false positive). + ``required_task_ids`` enables strict coverage for an explicit environment capability + set. Legitimate skips outside that set remain non-fatal but are always reported. """ label = (label or "local").strip() or "local" battery = battery_fingerprint(tasks) plane, _workspace_slug = make_plane_client() print(f"canary battery={battery} label={label} tasks={[task['id'] for task in tasks]}") - broken: list[str] = [] - verified_count = 0 - empty_agent = {"final_text": "", "calls": []} + broken_ids: list[str] = [] + verified_ids: list[str] = [] + skipped_reasons: dict[str, str] = {} + errored_reasons: dict[str, list[str]] = {} + + def record_error(task_id: str, reason: str) -> None: + errored_reasons.setdefault(task_id, []).append(reason) for task in tasks: + task_id = str(task["id"]) context: dict[str, Any] = {} task_needs = set(task.get("needs") or set()) + verifier_exercised = False try: try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + seed( + plane, + run_id=uuid.uuid4().hex, + needs=task_needs, + ctx=context, + task_id=task_id, + ) except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") + skipped_reasons[task_id] = skip.reason + print(f" {task_id} SKIPPED: {skip.reason}") + continue + except Exception as exc: + reason = f"infra_seed {type(exc).__name__}: {exc}" + record_error(task_id, reason) + print(f" {task_id} canary ERROR[infra_seed]: {exc}", file=sys.stderr) continue if "bug_type" in task_needs and not context.get("bug_type"): reason = context.get("bug_type_skip_reason") or "bug_type unavailable" - print(f" {task['id']} SKIPPED: {reason}") - continue - try: - ok, note = await task["verify"](plane, context, empty_agent) - except TaskSkipped as skip: - print(f" {task['id']} SKIPPED: {skip.reason}") + skipped_reasons[task_id] = str(reason) + print(f" {task_id} SKIPPED: {reason}") continue - verified_count += 1 - if ok: - broken.append(task["id"]) - print(f" BROKEN VERIFIER: {task['id']} note={note!r}") - else: - print(f" {task['id']} ok=False note={note!r}") + for probe_index, final_text in enumerate(canary_probe_texts(task_id)): + probe = { + "final_text": final_text, + "calls": [], + "call_source": "canary", + } + try: + ok, note = await task["verify"](plane, context, probe) + except TaskSkipped as skip: + skipped_reasons[task_id] = skip.reason + print(f" {task_id} SKIPPED during verifier: {skip.reason}") + break + verifier_exercised = True + if ok: + broken_ids.append(task_id) + print( + f" BROKEN VERIFIER: {task_id} accepted canary probe " + f"{probe_index} final_text={final_text!r} note={note!r}" + ) + break + print(f" {task_id} probe={probe_index} ok=False note={note!r}") + if task_id not in skipped_reasons: + verified_ids.append(task_id) except Exception as exc: - print(f" {task['id']} canary ERROR: {exc}", file=sys.stderr) - broken.append(task["id"]) + record_error(task_id, f"{type(exc).__name__}: {exc}") + print(f" {task_id} canary ERROR: {exc}", file=sys.stderr) finally: try: teardown(plane, context) except Exception as exc: - print(f" teardown error: {exc}", file=sys.stderr) + record_error(task_id, f"teardown {type(exc).__name__}: {exc}") + print(f" {task_id} teardown ERROR: {exc}", file=sys.stderr) - if broken: - for task_id in broken: + if ( + verifier_exercised + and task_id not in verified_ids + and task_id not in skipped_reasons + and task_id not in errored_reasons + ): + verified_ids.append(task_id) + + skipped_ids = list(skipped_reasons) + errored_ids = list(errored_reasons) + total = len(tasks) + print(f"canary coverage: verified={len(verified_ids)}/{total} ids={verified_ids}") + print(f"canary coverage: skipped={len(skipped_ids)} ids={skipped_ids} reasons={skipped_reasons}") + print(f"canary coverage: errored={len(errored_ids)} ids={errored_ids} reasons={errored_reasons}") + + required = set(required_task_ids or ()) + missing_required = sorted(required - set(verified_ids)) + if missing_required: + print( + f"canary strict coverage FAILED: missing required ids={missing_required}", + file=sys.stderr, + ) + + if broken_ids: + for task_id in broken_ids: print(f"BROKEN VERIFIER: {task_id}", file=sys.stderr) - return 1 - if verified_count == 0: + if not verified_ids: print( - "error: canary verified 0 tasks (all skipped by environment/fixture gates) " - "— nothing exercised; refusing exit 0", + "error: canary verified 0 tasks — nothing exercised; refusing exit 0", file=sys.stderr, ) + if broken_ids or errored_ids or missing_required or not verified_ids: return 1 - print(f"canary: all verifiers reject empty agent ({verified_count} verified)") + print(f"canary: verified zero-call probes rejected ({len(verified_ids)} verifier(s))") return 0 + + +__all__ = ["CANARY_CANNED_OUTPUTS", "canary_probe_texts", "run_canary"] diff --git a/evals/runner/live.py b/evals/runner/live.py index ce0349c3..8c5e6fb5 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -14,6 +14,9 @@ from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS +from evals.evidence import normalize_evidence_sentinels +from evals.report.load import load_rows +from evals.report.summary import completeness_statement, execution_coverage_statement, summarize from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown from evals.tasks.catalog import battery_fingerprint, task_author @@ -36,14 +39,6 @@ def _system_preamble(workspace_slug: str, project_name: str) -> str: ) -def classify_call(tool: str, optimal: set[str], alternate: set[str]) -> str: - if tool in optimal: - return "optimal" - if tool in alternate: - return "alternate" - return "out_of_set" - - def stdio_server_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: """Build MCP stdio env from scratch — never inherit os.environ (F6).""" environment: dict[str, str] = {} @@ -103,17 +98,12 @@ async def run_agent_task_via_driver( task: dict[str, Any], ctx: dict[str, Any], workspace_slug: str, - optimal_tools: set[str] | None = None, - alternate_tools: set[str] | None = None, server_env: dict[str, str] | None = None, ) -> TaskResult: """Run one task through the selected driver.""" project_name = ctx["project_name"] system = _system_preamble(workspace_slug, project_name) prompt = format_task_prompt(task, ctx, strict=True) - optimal = set(optimal_tools) if optimal_tools is not None else set(task["optimal_tools"]) - alternate = set(alternate_tools) if alternate_tools is not None else set(task["alternate_tools"]) - assert optimal.isdisjoint(alternate), f"{task['id']}: optimal/alternate overlap" mcp_env = stdio_server_env(extra=server_env) # Drivers are sync (CLI subprocess or API loop); keep them off this loop. @@ -125,13 +115,9 @@ async def run_agent_task_via_driver( MAX_ITERATIONS, system=system, cwd=Path(__file__).resolve().parent.parent.parent, + evidence_sentinels=ctx.get("evidence_sentinels"), ) - return agent_run_to_task_result( - agent_run, - optimal=optimal, - alternate=alternate, - classify=classify_call, - ) + return agent_run_to_task_result(agent_run) def _make_task_row( @@ -146,6 +132,7 @@ def _make_task_row( requested_tier: str | None, task: dict[str, Any], repetition: int, + expected_rows: int, battery: str, server: str, ) -> TaskResult: @@ -165,6 +152,7 @@ def _make_task_row( task_id=str(task["id"]), author=task_author(task), rep=repetition, + expected_rows=expected_rows, ) @@ -180,7 +168,17 @@ def _seed_fixtures( task_needs = set(task.get("needs") or set()) # Seed wrap: TaskSkipped → skip; other failures → infra_seed. try: - seed(plane, run_id=uuid.uuid4().hex, needs=task_needs, ctx=context) + seed( + plane, + run_id=uuid.uuid4().hex, + needs=task_needs, + ctx=context, + task_id=str(task["id"]), + ) + if "read" in set(task.get("tags") or set()) and not normalize_evidence_sentinels( + context.get("evidence_sentinels") + ): + raise RuntimeError(f"{task['id']} seed did not register target-entity evidence sentinels") except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason @@ -234,8 +232,6 @@ async def _drive_agent( task=task, ctx=context, workspace_slug=workspace_slug, - optimal_tools=set(task["optimal_tools"]), - alternate_tools=set(task["alternate_tools"]), server_env=server_env, ) except PromptBindError as exc: @@ -273,7 +269,6 @@ def _apply_agent_run( model_alias: str, requested_tier: str | None, model_id: str | None, - external: bool, ) -> None: """Copy agent metrics and restore the run-level model and server identity.""" row.apply_agent_result(agent) @@ -282,10 +277,6 @@ def _apply_agent_run( row.requested_model = model_alias row.requested_tier = requested_tier row.resolved_model = model_id - if external: - # Foreign tool names are not comparable to our catalog. - row.alternate_calls = None - row.out_of_set_calls = None def _record_cli_infra_stop( @@ -338,6 +329,10 @@ async def _verify_task( { "final_text": agent.final_text, "calls": agent_row["calls"], + "call_source": agent.call_source, + "evidence_trace_available": agent.evidence_trace_available, + "driver_notes": list(agent.driver_notes), + "result_pair_mismatch": agent.result_pair_mismatch, }, ) row.success = bool(ok) @@ -382,11 +377,12 @@ def _record_unexpected( ) -def _remove_fixtures(plane: Any, context: dict[str, Any]) -> None: +def _remove_fixtures(plane: Any, context: dict[str, Any], row: TaskResult) -> None: """Remove task fixtures and retain the historical teardown diagnostics.""" try: teardown(plane, context) except Exception as exc: + row.cleanup_error = f"{type(exc).__name__}: {exc}" print(f" teardown error: {exc}", file=sys.stderr) if context.get("project_name"): print(f" orphaned project: {context['project_name']}", file=sys.stderr) @@ -399,6 +395,7 @@ async def _run_task_repetition( workspace_slug: str, task: dict[str, Any], repetition: int, + expected_rows: int, run_id: str, git_revision: str, label: str, @@ -425,6 +422,7 @@ async def _run_task_repetition( requested_tier=requested_tier, task=task, repetition=repetition, + expected_rows=expected_rows, battery=battery, server="external" if external else "local", ) @@ -448,7 +446,6 @@ async def _run_task_repetition( model_alias=model_alias, requested_tier=requested_tier, model_id=model_id, - external=external, ) if not _record_cli_infra_stop( row, @@ -462,7 +459,7 @@ async def _run_task_repetition( # Anything outside seed/driver/verify wraps. _record_unexpected(row, exc, task=task, repetition=repetition, context=context) finally: - _remove_fixtures(plane, context) + _remove_fixtures(plane, context, row) return row @@ -500,6 +497,7 @@ async def run_live( run_id = uuid.uuid4().hex git_revision = read_git_revision() battery = battery_fingerprint(tasks) + total_runs = len(tasks) * reps out_path.parent.mkdir(parents=True, exist_ok=True) resume_skip: set[tuple[str, int, str]] = set() @@ -531,6 +529,7 @@ async def run_live( driver=driver_name, provider=provider_id, git_sha=git_revision, + expected_rows=total_runs, ) if maybe_write_run_meta(out_path, meta): print(f"wrote meta header battery={battery} label={label}", flush=True) @@ -559,7 +558,6 @@ async def run_live( # A battery is tens of minutes of silence otherwise: one line before each # repetition says what is running now, and one after says where the run is. - total_runs = len(tasks) * reps started_at = time.monotonic() finished = 0 passed = 0 @@ -584,6 +582,7 @@ async def run_live( workspace_slug=workspace_slug, task=task, repetition=repetition, + expected_rows=total_runs, run_id=run_id, git_revision=git_revision, label=label, @@ -617,4 +616,21 @@ async def run_live( f"finished {finished}/{total_runs} in {_elapsed(started_at)}: {passed} pass, {failed} fail, {skipped} skip", flush=True, ) - return 0 + selected_task_ids = {str(task["id"]) for task in tasks} + result_rows = [ + row + for row in load_rows(out_path) + if row.label == label + and (not row.battery or row.battery == battery) + and row.task_id in selected_task_ids + and 0 <= row.rep < reps + ] + summary = summarize(result_rows, expected_rows=total_runs) + if summary.aggregate_n: + rate = summary.aggregate_k / summary.aggregate_n + print(f"success: {summary.aggregate_k}/{summary.aggregate_n} ({rate:.1%})", flush=True) + else: + print("success: 0/0 (n/a; no evaluated rows)", flush=True) + print(execution_coverage_statement(summary), flush=True) + print(completeness_statement(summary), flush=True) + return 0 if summary.complete else 1 diff --git a/evals/runner/meta.py b/evals/runner/meta.py index 231555f3..9cd02855 100644 --- a/evals/runner/meta.py +++ b/evals/runner/meta.py @@ -46,10 +46,11 @@ def make_run_meta_row( requested_model: str | None = None, requested_tier: str | None = None, resolved_model: str | None = None, + expected_rows: int | None = None, ts: str | None = None, ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" - return { + row = { "schema_version": RESULT_SCHEMA_VERSION, "row_type": "meta", "run_id": run_id, @@ -65,6 +66,9 @@ def make_run_meta_row( "git_sha": git_sha, "ts": ts or datetime.now(timezone.utc).isoformat(), } + if expected_rows is not None: + row["expected_rows"] = expected_rows + return row def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: diff --git a/evals/runner/resume.py b/evals/runner/resume.py index 197aee00..044a793d 100644 --- a/evals/runner/resume.py +++ b/evals/runner/resume.py @@ -8,6 +8,7 @@ from typing import Any from evals.results import TaskResult +from evals.skip_taxonomy import is_expected_environment_capability_skip from .meta import is_meta_or_non_task_row @@ -15,9 +16,9 @@ def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: """Return True if a prior row is a completed result that resume should skip. - Re-run when ``error_class`` starts with ``infra_`` or when ``error`` is non-null. - Rows with ``skipped`` set are treated as complete and are not retried (intentional: - environment/fixture skips are stable outcomes, not infra failures). + Re-run rows with errors, cleanup failures, or unexpected skips. Known missing + environment capabilities are legitimate terminal outcomes because rerunning cannot + add them; fixture collisions and unknown skips may be repairable. Pure function — unit-tested without the live battery. """ result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) @@ -26,6 +27,10 @@ def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: return False if result.error is not None: return False + if result.cleanup_error is not None: + return False + if result.skipped is not None: + return is_expected_environment_capability_skip(result.skipped) return True diff --git a/evals/skip_taxonomy.py b/evals/skip_taxonomy.py new file mode 100644 index 00000000..22ea32b3 --- /dev/null +++ b/evals/skip_taxonomy.py @@ -0,0 +1,79 @@ +"""Explicit run-completeness taxonomy for task skip reasons. + +Known missing environment capabilities are expected skips: they reduce execution +coverage but do not make an otherwise clean run incomplete. A dirty environment that +requires operator cleanup, such as a fixture collision, is unexpected. Unknown reasons +are also unexpected by default; there is deliberately no ``env:*`` catch-all. Plan-gate +reasons must name one of the explicitly supported capabilities below, and the activity +worker reason must match exactly. +""" + +from __future__ import annotations + +from typing import Literal + +SkipDisposition = Literal["expected-capability", "dirty-environment", "unexpected"] + +PLAN_GATED_PREFIX = "env:plan-gated:" +NO_ACTIVITY_WORKER_REASON = "env:no-activity-worker" +FIXTURE_COLLISION_PREFIX = "env:fixture-collision:" + +# Derived from the plan-gated seed surfaces: customer and release fixture seeders, the +# work-item-type seeder, and the initiative/teamspace plan refusals characterized by +# seed.projects.is_plan_gate. Keep this closed: a new capability is unexpected until its +# actual gate site is reviewed and added deliberately. +PLAN_GATED_CAPABILITIES = frozenset( + { + "customers", + "initiatives", + "releases", + "teamspaces", + "work-item-types", + } +) + + +def _plan_gated_capability(reason: str) -> str | None: + if not reason.startswith(PLAN_GATED_PREFIX): + return None + capability = reason.removeprefix(PLAN_GATED_PREFIX) + return capability if capability in PLAN_GATED_CAPABILITIES else None + + +def classify_skip_reason(reason: str) -> SkipDisposition: + """Classify a known capability skip, dirty environment, or unknown reason.""" + if _plan_gated_capability(reason) is not None: + return "expected-capability" + if reason == NO_ACTIVITY_WORKER_REASON: + return "expected-capability" + if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): + return "dirty-environment" + return "unexpected" + + +def is_expected_environment_capability_skip(reason: str) -> bool: + """Return whether a known absent environment capability caused the skip.""" + return classify_skip_reason(reason) == "expected-capability" + + +def skip_reason_family(reason: str) -> str: + """Return the stable reporting family for a skip reason.""" + if _plan_gated_capability(reason) is not None: + return "plan-gated" + if reason == NO_ACTIVITY_WORKER_REASON: + return "no-activity-worker" + if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): + return "fixture-collision" + return reason or "" + + +__all__ = [ + "FIXTURE_COLLISION_PREFIX", + "NO_ACTIVITY_WORKER_REASON", + "PLAN_GATED_CAPABILITIES", + "PLAN_GATED_PREFIX", + "SkipDisposition", + "classify_skip_reason", + "is_expected_environment_capability_skip", + "skip_reason_family", +] diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py index f70a8ed6..8db1c690 100644 --- a/tests/evals/report/test_compare.py +++ b/tests/evals/report/test_compare.py @@ -1,82 +1,92 @@ -"""Offline eval tests for compare.""" +"""Offline eval tests for paired comparisons.""" from __future__ import annotations import math +from pathlib import Path import pytest from evals.report import ( ab_compare, - sign_test_pvalue, + paired_bootstrap_mean_ci, + paired_permutation_pvalue, + print_ab_report, ) -def test_sign_test_behaviours(): - def test_sign_test_all_positive_hand_computed(): - deltas = [1.0, 2.0, 3.0, 0.5, 4.0] - p = sign_test_pvalue(deltas) - assert p == pytest.approx(2.0 * (1.0 / 32.0)) - assert p == pytest.approx(0.0625) - - def test_sign_test_four_of_five_hand_computed(): - deltas = [1.0, 1.0, 1.0, 1.0, -1.0] - p = sign_test_pvalue(deltas) - right = (math.comb(5, 4) + math.comb(5, 5)) / 32.0 - assert p == pytest.approx(2.0 * right) - assert p == pytest.approx(0.375) - - def test_sign_test_drops_zeros_and_none_when_empty(): - assert sign_test_pvalue([0.0, 0.0]) is None - assert sign_test_pvalue([]) is None - # One positive, one zero → n=1, k=1 → p = 2*(1/2) = 1.0 - assert sign_test_pvalue([3.0, 0.0]) == pytest.approx(1.0) - - test_sign_test_all_positive_hand_computed() - test_sign_test_four_of_five_hand_computed() - test_sign_test_drops_zeros_and_none_when_empty() - - -def test_ab_compare_behaviours(): - def test_ab_compare_paired_deltas_and_sign_test(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, - {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, # not paired - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, # delta -3 - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, # delta +1 - {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, # A failed → not paired - ] - cmp = ab_compare(rows_a, rows_b) - assert cmp["n_paired"] == 2 - deltas = {p["task_id"]: p["delta"] for p in cmp["paired_tasks"]} - assert deltas["R1"] == -3.0 - assert deltas["R2"] == 1.0 - assert cmp["median_delta"] == pytest.approx(-1.0) # median of [-3, 1] - assert cmp["sign_test_p"] is not None - assert cmp["success_a"]["k"] == 2 and cmp["success_a"]["n"] == 3 - assert cmp["success_b"]["k"] == 3 and cmp["success_b"]["n"] == 3 - - def test_ab_compare_multi_rep_uses_median_successful_call_counts(): - rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, - ] - rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, - ] - - cmp = ab_compare(rows_a, rows_b) - - assert cmp["multi_rep"] is True - assert cmp["unstable_a"] == 1 - assert cmp["unstable_b"] == 0 - assert cmp["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] - - test_ab_compare_paired_deltas_and_sign_test() - test_ab_compare_multi_rep_uses_median_successful_call_counts() +def test_paired_permutation_retains_ties_and_uses_delta_magnitudes(): + assert paired_permutation_pvalue([]) is None + assert paired_permutation_pvalue([0.0, 0.0]) == 1.0 + + # A sign test sees 6 positive versus 10 negative deltas and cannot detect + # the coherent large-magnitude shift. The paired randomization distribution + # uses those magnitudes while all 30 zero-delta task pairs remain in n=46. + deltas = [10.0] * 6 + [-0.1] * 10 + [0.0] * 30 + permutation_p = paired_permutation_pvalue(deltas) + old_sign_p = 2 * sum(math.comb(16, index) for index in range(7)) / (2**16) + assert old_sign_p > 0.05 + assert permutation_p == pytest.approx(0.03125) + + +def test_paired_bootstrap_small_sample_is_task_paired_and_wide(): + deltas = [1.0, 1.0, -1.0, 0.0, 0.0] + + lower, upper = paired_bootstrap_mean_ci(deltas) + + assert lower is not None and upper is not None + assert lower < 0.0 < upper + assert upper - lower >= 1.0 + + +def test_ab_compare_behaviours(capsys): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["n_paired"] == 2 # R3 has no successful A call count + deltas = {pair["task_id"]: pair["delta"] for pair in comparison["paired_tasks"]} + assert deltas == {"R1": -3.0, "R2": 1.0} + assert comparison["mean_delta"] == pytest.approx(-1.0) + assert comparison["median_delta"] == pytest.approx(-1.0) + assert comparison["call_permutation_p"] is not None + assert comparison["call_zero_deltas"] == 0 + assert comparison["n_paired_success"] == 3 + assert comparison["paired_success_delta"] == pytest.approx(1 / 3) + assert comparison["success_a"]["k"] == 2 and comparison["success_a"]["n"] == 3 + assert comparison["success_b"]["k"] == 3 and comparison["success_b"]["n"] == 3 + + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + output = capsys.readouterr().out + assert "paired-bootstrap95" in output + assert "paired permutation p-value" in output + assert "zero-delta ties retained" in output + assert "sign-test" not in output + assert "noise floor" not in output + + +def test_ab_compare_multi_rep_uses_median_successful_call_counts(): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["multi_rep"] is True + assert comparison["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] diff --git a/tests/evals/report/test_load.py b/tests/evals/report/test_load.py index 28465b0d..89724e0a 100644 --- a/tests/evals/report/test_load.py +++ b/tests/evals/report/test_load.py @@ -5,12 +5,15 @@ import json from pathlib import Path +import pytest + from evals.report import ( dedupe_rows_latest, is_infra_error_row, load_rows, summarize, ) +from tests.evals.conftest import case_params def test_is_infra_error_row_covers_infrastructure_prefix(): @@ -18,83 +21,88 @@ def test_is_infra_error_row_covers_infrastructure_prefix(): assert is_infra_error_row({"error_class": "task"}) is False -def test_load_behaviours(tmp_path, capsys): - def test_load_rows_dedupe_latest_wins(tmp_path): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p) # default dedupe=latest - assert len(loaded) == 1 - assert loaded[0].num_calls == 9 - assert loaded[0].success is False - - def test_load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path, capsys): - p = tmp_path / "dup.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "success": True}, - {"task_id": "R1", "rep": 0, "label": "local", "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - loaded = load_rows(p, dedupe="none") - assert len(loaded) == 2 - err = capsys.readouterr().err - assert "duplicate" in err - assert "R1" in err - - def test_load_rows_skips_meta_and_missing_task_id(tmp_path): - p = tmp_path / "r.jsonl" - lines = [ - json.dumps( - { - "row_type": "meta", - "run_id": "abc", - "label": "candidate", - "battery": "deadbeef0001", - "model": "sonnet", - "driver": "claude-cli", - "git_sha": "x", - "ts": "t", - } - ), - json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id - json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), - ] - p.write_text("\n".join(lines) + "\n", encoding="utf-8") - rows = load_rows(p) - assert len(rows) == 1 - assert rows[0].task_id == "R1" - - _d0 = tmp_path / "test_load_rows_dedupe_latest_wins" - _d0.mkdir() - test_load_rows_dedupe_latest_wins(_d0) - _d1 = tmp_path / "test_load_rows_no_dedupe_warns_on_duplicate_keys" - _d1.mkdir() - test_load_rows_no_dedupe_warns_on_duplicate_keys(_d1, capsys) - _d2 = tmp_path / "test_load_rows_skips_meta_and_missing_task_id" - _d2.mkdir() - test_load_rows_skips_meta_and_missing_task_id(_d2) - - -def test_real_historical_rows_parse_and_report_with_backward_defaults(): - fixture = Path(__file__).parents[2] / "fixtures" / "evals_historical_rows.jsonl" +def _load_rows_dedupe_latest_wins(tmp_path, _capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p) # default dedupe=latest + assert len(loaded) == 1 + assert loaded[0].num_calls == 9 + assert loaded[0].success is False + + +def _load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path, capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p, dedupe="none") + assert len(loaded) == 2 + err = capsys.readouterr().err + assert "duplicate" in err + assert "R1" in err + + +def _load_rows_skips_meta_and_surfaces_missing_task_id(tmp_path, capsys): + p = tmp_path / "r.jsonl" + lines = [ + json.dumps( + { + "row_type": "meta", + "run_id": "abc", + "label": "candidate", + "battery": "deadbeef0001", + "model": "sonnet", + "driver": "claude-cli", + "git_sha": "x", + "ts": "t", + } + ), + json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + rows = load_rows(p) + assert len(rows) == 2 + assert rows[0].error_class == "harness_report_load" + assert rows[1].task_id == "R1" + assert "recording result without task_id as a harness error" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "case", + case_params( + _load_rows_dedupe_latest_wins, + _load_rows_no_dedupe_warns_on_duplicate_keys, + _load_rows_skips_meta_and_surfaces_missing_task_id, + ), +) +def test_load_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) + + +def test_schema_v0_rows_parse_with_backward_defaults(): + """Synthetic schema-0 rows retain defaults unrelated to this change.""" + fixture = Path(__file__).parents[2] / "fixtures" / "evals_schema_v0_rows.jsonl" rows = load_rows(fixture) assert [row.schema_version for row in rows] == [0, 0] by_task = {row.task_id: row for row in rows} - battery4 = by_task["L3"] - assert battery4.final_text == "" - assert battery4.result_tokens_estimated is None - assert battery4.alternate_calls is None - assert battery4.calls[0].result_tokens is None - assert battery4.calls[0].action == "create" - - battery5 = by_task["R2"] - assert battery5.final_text.endswith("\n4") - assert battery5.result_tokens_estimated is True - assert [call.result_tokens for call in battery5.calls] == [315, 64] + release_row = by_task["L3"] + assert release_row.final_text == "" + assert release_row.result_tokens_estimated is None + assert release_row.calls[0].result_tokens is None + assert release_row.calls[0].action == "create" + + count_row = by_task["R2"] + assert count_row.final_text.endswith("\n4") + assert count_row.result_tokens_estimated is True + assert [call.result_tokens for call in count_row.calls] == [315, 64] summary = summarize(rows) assert summary.tasks["L3"].success == "1/1" @@ -116,3 +124,30 @@ def test_dedupe_rows_latest_pure(): by_id = {r.task_id: r for r in out} assert by_id["R1"].num_calls == 5 assert by_id["R2"].num_calls == 3 + + +def test_malformed_rows_surface_as_completeness_errors(tmp_path, capsys): + path = tmp_path / "malformed.jsonl" + path.write_text( + "\n".join( + [ + json.dumps({"task_id": "R1", "success": True, "calls": []}), + "{not-json", + json.dumps(["not", "an", "object"]), + ] + ) + + "\n", + encoding="utf-8", + ) + + rows = load_rows(path) + summary = summarize(rows, expected_rows=3) + + assert len(rows) == 3 + assert summary.aggregate_n == 1 + assert summary.harness_errors == 2 + assert summary.complete is False + assert {row.error_class for row in rows if row.error} == {"harness_report_load"} + warnings = capsys.readouterr().err + assert "recording invalid JSON as a harness error" in warnings + assert "recording non-object JSON as a harness error" in warnings diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 06eb8d19..85c221b0 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -9,68 +9,235 @@ from evals import report as report_mod from evals.report import ( + completeness_statement, + execution_coverage_statement, + format_tool_distribution, + format_tool_variability, is_infra_error_row, load_rows, summarize, wilson_interval, ) +from tests.evals.conftest import case_params -def test_summarize_behaviours(): - def test_summarize_excludes_infra_errors_from_success(): - rows = [ - {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "HttpError: 409", - "error_class": "infra_seed", - }, - { - "task_id": "R1", - "success": False, - "num_calls": 0, - "calls": [], - "error": "timeout after 120s", - "error_class": "infra_cli", - }, - {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, - ] - summary = summarize(rows) - assert summary.infra_errors == 2 - assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows - assert summary.tasks["R1"].k == 1 - assert summary.tasks["R1"].success == "1/2" - assert summary.tasks["R1"].infra_err == 2 - assert is_infra_error_row(rows[1]) is True - assert is_infra_error_row(rows[0]) is False - - def test_summarize_aggregate_wilson_and_call_variance(): - rows = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - ] - s = summarize(rows) - assert s.tasks["R1"].n == 3 - assert s.tasks["R1"].k == 2 - assert s.tasks["R1"].calls_min == 2.0 - assert s.tasks["R1"].calls_max == 6.0 - assert s.tasks["R1"].med_calls == 4.0 - assert s.tasks["R1"].unstable is True - assert s.tasks["R2"].unstable is False - assert s.aggregate_k == 3 - assert s.aggregate_n == 4 - assert s.multi_rep is True - assert s.unstable_task_ids == ["R1"] - assert s.unstable_tasks == 1 - assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 - - test_summarize_excludes_infra_errors_from_success() - test_summarize_aggregate_wilson_and_call_variance() +def test_completeness_is_independent_from_success_rate(): + complete = summarize( + [ + {"task_id": "R1", "success": True, "calls": []}, + {"task_id": "R2", "success": False, "calls": []}, + {"task_id": "C1", "skipped": "env:plan-gated:customers", "calls": []}, + ], + expected_rows=3, + ) + assert complete.aggregate_k == 1 + assert complete.aggregate_n == 2 + assert complete.completed_rows == 3 + assert complete.expected_skips == 1 + assert complete.complete is True + assert completeness_statement(complete).startswith("RUN COMPLETE:") + assert execution_coverage_statement(complete) == ( + "EXECUTION COVERAGE: 2/3 rows evaluated (66.7%); skipped tasks=[C1 (env:plan-gated:customers)]" + ) + + missing_worker = summarize( + [{"task_id": "L2", "skipped": "env:no-activity-worker", "calls": []}], + expected_rows=1, + ) + assert missing_worker.aggregate_n == 0 + assert missing_worker.completed_rows == 1 + assert missing_worker.expected_skips == 1 + assert missing_worker.expected_skip_reasons == {"no-activity-worker": 1} + assert missing_worker.complete is True + assert completeness_statement(missing_worker).startswith("RUN COMPLETE:") + + verifier_crash = summarize( + [ + {"task_id": "R1", "success": True, "calls": []}, + {"task_id": "W6", "error": "RuntimeError: verifier broke", "error_class": "task", "calls": []}, + ], + expected_rows=2, + ) + assert verifier_crash.aggregate_k == 1 + assert verifier_crash.aggregate_n == 1 + assert verifier_crash.harness_errors == 1 + assert verifier_crash.complete is False + assert completeness_statement(verifier_crash).startswith("RUN INCOMPLETE:") + + collisions = summarize( + [ + {"task_id": "R1", "skipped": "env:fixture-collision:customers:Acme", "calls": []}, + {"task_id": "R2", "skipped": "env:fixture-collision:release_tags:eval-rc1", "calls": []}, + ], + expected_rows=2, + ) + assert collisions.completed_rows == 0 + assert collisions.unexpected_skips == 2 + assert collisions.unexpected_skip_reasons == {"fixture-collision": 2} + assert collisions.complete is False + + unknown = summarize( + [{"task_id": "L2", "skipped": "env:new-reason", "calls": []}], + expected_rows=1, + ) + assert unknown.unexpected_skips == 1 + assert unknown.unexpected_skip_reasons == {"env:new-reason": 1} + assert unknown.complete is False + + cleanup = summarize( + [{"task_id": "R1", "success": True, "cleanup_error": "RuntimeError: delete failed", "calls": []}], + expected_rows=1, + ) + assert cleanup.aggregate_k == cleanup.aggregate_n == 1 + assert cleanup.cleanup_errors == 1 + assert cleanup.complete is False + + +def _summarize_excludes_infra_errors_from_success(): + rows = [ + {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "HttpError: 409", + "error_class": "infra_seed", + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "timeout after 120s", + "error_class": "infra_cli", + }, + {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, + ] + summary = summarize(rows) + assert summary.infra_errors == 2 + assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows + assert summary.tasks["R1"].k == 1 + assert summary.tasks["R1"].success == "1/2" + assert summary.tasks["R1"].infra_err == 2 + assert summary.tasks["R1"].failed_tool_reps == 3 + assert "failed excluded=3" in format_tool_distribution(summary.tasks["R1"]) + assert is_infra_error_row(rows[1]) is True + assert is_infra_error_row(rows[0]) is False + + +def _summarize_aggregate_wilson_and_call_variance(): + rows = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + ] + s = summarize(rows) + assert s.tasks["R1"].n == 3 + assert s.tasks["R1"].k == 2 + assert s.tasks["R1"].calls_min == 2.0 + assert s.tasks["R1"].calls_q1 == 2.5 + assert s.tasks["R1"].med_calls == 3.0 + assert s.tasks["R1"].calls_q3 == 3.5 + assert s.tasks["R1"].calls_max == 4.0 + assert s.tasks["R1"].unstable is True + assert s.tasks["R2"].unstable is False + assert s.aggregate_k == 3 + assert s.aggregate_n == 4 + assert s.multi_rep is True + assert s.unstable_task_ids == ["R1"] + assert s.unstable_tasks == 1 + assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 + + +def _tool_distribution_uses_successful_repetitions(): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 3, + "calls": [{"tool": "a"}, {"tool": "a"}, {"tool": "b"}], + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 2, + "calls": [{"tool": "a"}, {"tool": "c"}], + }, + { + "task_id": "R1", + "rep": 2, + "success": False, + "num_calls": 1, + "calls": [{"tool": "failed_only"}], + }, + { + "task_id": "R2", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"tool": "one_rep"}], + }, + { + "task_id": "R3", + "rep": 0, + "success": False, + "num_calls": 1, + "calls": [{"tool": "failed_only"}], + }, + {"task_id": "R4", "rep": 0, "skipped": "unavailable", "calls": []}, + ] + + summary = summarize(rows) + r1 = summary.tasks["R1"] + assert r1.calls_min == 2.0 # failed one-call repetition is not an observed successful floor + assert r1.calls_q1 == 2.25 + assert r1.med_calls == 2.5 + assert r1.calls_q3 == 2.75 + assert r1.calls_max == 3.0 + assert r1.calls_min <= r1.calls_q1 <= r1.med_calls <= r1.calls_q3 <= r1.calls_max + assert r1.tool_reps == 2 + assert r1.failed_tool_reps == 1 + assert r1.tool_rep_frequency == {"a": 1.0, "b": 0.5, "c": 0.5} + assert r1.tool_call_counts == {"a": 3, "b": 1, "c": 1} + assert "failed_only" not in r1.tool_call_counts + assert r1.variable_tool_names == ["b", "c"] + assert summary.variable_tool_tasks == 1 + assert summary.total_tasks == 4 + assert format_tool_variability(summary) == "1/4 tasks" + r1_distribution = format_tool_distribution(r1) + assert "success-only n=2; failed excluded=1" in r1_distribution + assert "core:a(3c)" in r1_distribution + assert "variable:b=50%(1c),c=50%(1c)" in r1_distribution + + r2 = summary.tasks["R2"] + assert r2.tool_reps == 1 + assert r2.failed_tool_reps == 0 + assert r2.tool_rep_frequency == {} + assert r2.tool_call_counts == {"one_rep": 1} + assert format_tool_distribution(r2) == "success-only n=1; failed excluded=0; frequency=—" + + r3 = summary.tasks["R3"] + assert r3.tool_reps == 0 + assert r3.failed_tool_reps == 1 + assert r3.tool_rep_frequency == {} + assert r3.tool_call_counts == {} + assert format_tool_distribution(r3) == "success-only n=0; failed excluded=1; frequency=—" + + +@pytest.mark.parametrize( + "case", + case_params( + _summarize_excludes_infra_errors_from_success, + _summarize_aggregate_wilson_and_call_variance, + _tool_distribution_uses_successful_repetitions, + ), +) +def test_summarize_behaviours(case): + case() def test_wilson_interval_bounds(): @@ -83,7 +250,7 @@ def test_wilson_interval_bounds(): assert wilson_interval(0, 0) == (0.0, 0.0) -def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_path: Path, capsys): +def test_multi_rep_synthetic_file_reports_wilson_and_instability_without_noise_claim(tmp_path: Path, capsys): path = tmp_path / "multi.jsonl" outcomes = { "R1": [True, True, True], @@ -125,11 +292,10 @@ def test_multi_rep_synthetic_file_reports_wilson_unstable_and_noise_floor(tmp_pa assert "2/3" in r2_line assert "[0.21,0.94]" in r2_line assert "YES" in r2_line - assert "measured noise floor: 1 task flipped at least once" in output - assert "minimum meaningful difference: 2 tasks" in output + assert "noise floor" not in output -def test_single_rep_summary_rendering_is_unchanged(capsys): +def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): rows = [{"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 2, "calls": []}] report_mod.print_table(summarize(rows), "Summary: sample.jsonl") @@ -137,9 +303,13 @@ def test_single_rep_summary_rendering_is_unchanged(capsys): assert capsys.readouterr().out == ( "Summary: sample.jsonl\n" "aggregate success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" - "task n success wilson95 med_calls opt IQR mispick err capped h_err i_err " - "med_rtok p95_rtok med_cum_in\n" - "-------------------------------------------------------------------------------------------------------------------------------\n" - "R1 1 1/1 [0.21,1.00] 2.0 1 2.0-2.0 0.0% 0 0 0 0 " - "- - 0\n" + "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "RUN COMPLETE: 1/1 rows completed\n" + "tool variability: —\n" + "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " + "err capped h_err i_err " + "med_rtok p95_rtok med_cum_in tool distribution\n" + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------\n" + "R1 1 1/1 [0.21,1.00] 2.0 2.0 2.0-2.0 0 0 " + " 0 0 - - 0 success-only n=1; failed excluded=0; frequency=—\n" ) diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 66f18420..408d5d58 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -5,6 +5,8 @@ import json from typing import Any +import pytest + from evals import report as report_mod from evals.report import ( build_multi_surface_table, @@ -12,6 +14,7 @@ render_multi_surface_table, summarize, ) +from tests.evals.conftest import case_params def _synth_row( @@ -20,13 +23,12 @@ def _synth_row( rep: int = 0, success: bool = True, num_calls: int = 2, - alt: int | None = 0, - oos: int | None = 0, server: str = "local", skipped: str | None = None, error: str | None = None, error_class: str | None = None, label: str = "local", + calls: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: return { "task_id": tid, @@ -34,13 +36,11 @@ def _synth_row( "label": label, "success": success, "num_calls": num_calls, - "alternate_calls": alt, - "out_of_set_calls": oos, "server": server, "skipped": skipped, "error": error, "error_class": error_class, - "calls": [], + "calls": list(calls or []), } @@ -61,132 +61,178 @@ def test_print_table_shows_infra_errors(capsys): assert " 2" in out # i_err column value -def test_report_behaviours(capsys, tmp_path): - def test_report_marks_entirely_estimated_result_token_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - } - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "estimated" - assert summary.tasks["R1"].result_tokens_mode == "estimated" - - report_mod.print_table(summary, "estimated") - output = capsys.readouterr().out - assert "entirely estimated" in output - assert "med_rtok~" in output - assert "~12" in output - - def test_report_marks_mixed_measured_and_estimated_columns(capsys): - rows = [ - { - "task_id": "R1", - "rep": 0, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], - "result_tokens_estimated": False, - }, - { - "task_id": "R1", - "rep": 1, - "success": True, - "num_calls": 1, - "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], - "result_tokens_estimated": True, - }, - ] - summary = summarize(rows) - assert summary.result_tokens_mode == "mixed" - assert summary.tasks["R1"].result_tokens_mode == "mixed" - - report_mod.print_table(summary, "mixed") - output = capsys.readouterr().out - assert "mixed measured and estimated" in output - assert "med_rtok*" in output - - def test_report_main_table_cli(tmp_path, capsys): - f1 = tmp_path / "a.jsonl" - f2 = tmp_path / "b.jsonl" - f1.write_text( - json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", - encoding="utf-8", - ) - f2.write_text( - json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", - encoding="utf-8", - ) - rc = report_mod.main(["--table", str(f1), str(f2)]) - assert rc == 0 - out = capsys.readouterr().out - assert "local" in out and "candidate" in out - assert "R1" in out - - def test_report_main_table_warns_when_battery_fingerprints_differ(tmp_path, capsys): - f1 = tmp_path / "old.jsonl" - f2 = tmp_path / "new.jsonl" - f1.write_text( - json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", - encoding="utf-8", +def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_path, capsys): + collision = tmp_path / "collision.jsonl" + collision.write_text( + "\n".join( + [ + json.dumps({"row_type": "meta", "expected_rows": 1}), + json.dumps(_synth_row("R1", skipped="env:fixture-collision:customers:Acme")), + ] ) - f2.write_text( - json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", - encoding="utf-8", + + "\n", + encoding="utf-8", + ) + + assert report_mod.main([str(collision)]) == 1 + collision_output = capsys.readouterr().out + assert "aggregate success: 0/0" in collision_output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in collision_output + assert "R1 (env:fixture-collision:customers:Acme)" in collision_output + assert "RUN INCOMPLETE:" in collision_output + assert "unexpected skips=1 [fixture-collision=1]" in collision_output + + plan_gated = tmp_path / "plan-gated.jsonl" + plan_gated.write_text( + "\n".join( + [ + json.dumps({"row_type": "meta", "expected_rows": 1}), + json.dumps(_synth_row("C1", skipped="env:plan-gated:customers")), + ] ) + + "\n", + encoding="utf-8", + ) - rc = report_mod.main(["--table", str(f1), str(f2)]) - - assert rc == 0 - captured = capsys.readouterr() - assert "spans battery fingerprints" in captured.err - assert "different task prompts/questions" in captured.err - - def test_report_main_markdown_flag(tmp_path, capsys): - f1 = tmp_path / "a.jsonl" - f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") - rc = report_mod.main(["--table", "--markdown", str(f1)]) - assert rc == 0 - out = capsys.readouterr().out - assert out.startswith("| task |") - assert "| R1 |" in out - assert "---" in out - - def test_report_main_no_dedupe_flag(tmp_path, capsys): - p = tmp_path / "d.jsonl" - rows = [ - _synth_row("R1", label="local", num_calls=1, success=True), - {**_synth_row("R1", label="local", num_calls=9, success=False)}, - ] - # Both rows have the same (task_id, rep, label), so latest-wins keeps one. - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - rc = report_mod.main(["--no-dedupe", str(p)]) - assert rc == 0 - # With no-dedupe, both rows enter summarize → n=2 for R1. - # (dedupe default would leave n=1.) - out = capsys.readouterr().out - assert "R1" in out - assert "2/2" in out or "1/2" in out # one success of two - - test_report_marks_entirely_estimated_result_token_columns(capsys) - test_report_marks_mixed_measured_and_estimated_columns(capsys) - _d2 = tmp_path / "test_report_main_table_cli" - _d2.mkdir() - test_report_main_table_cli(_d2, capsys) - _d3 = tmp_path / "test_report_main_table_warns_when_battery_fingerprints_differ" - _d3.mkdir() - test_report_main_table_warns_when_battery_fingerprints_differ(_d3, capsys) - _d4 = tmp_path / "test_report_main_markdown_flag" - _d4.mkdir() - test_report_main_markdown_flag(_d4, capsys) - _d5 = tmp_path / "test_report_main_no_dedupe_flag" - _d5.mkdir() - test_report_main_no_dedupe_flag(_d5, capsys) + assert report_mod.main([str(plan_gated)]) == 0 + plan_output = capsys.readouterr().out + assert "aggregate success: 0/0" in plan_output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in plan_output + assert "C1 (env:plan-gated:customers)" in plan_output + assert "RUN COMPLETE:" in plan_output + assert "expected skips=1 [plan-gated=1]" in plan_output + + +def _report_marks_entirely_estimated_result_token_columns(_tmp_path, capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + } + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "estimated" + assert summary.tasks["R1"].result_tokens_mode == "estimated" + + report_mod.print_table(summary, "estimated") + output = capsys.readouterr().out + assert "entirely estimated" in output + assert "med_rtok~" in output + assert "~12" in output + + +def _report_marks_mixed_measured_and_estimated_columns(_tmp_path, capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], + "result_tokens_estimated": False, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 1, + "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + }, + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "mixed" + assert summary.tasks["R1"].result_tokens_mode == "mixed" + + report_mod.print_table(summary, "mixed") + output = capsys.readouterr().out + assert "mixed measured and estimated" in output + assert "med_rtok*" in output + + +def _report_main_table_cli(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + f2 = tmp_path / "b.jsonl" + f1.write_text( + json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", + encoding="utf-8", + ) + rc = report_mod.main(["--table", str(f1), str(f2)]) + assert rc == 0 + out = capsys.readouterr().out + assert "local" in out and "candidate" in out + assert "R1" in out + + +def _report_main_table_warns_when_battery_fingerprints_differ(tmp_path, capsys): + f1 = tmp_path / "old.jsonl" + f2 = tmp_path / "new.jsonl" + f1.write_text( + json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", + encoding="utf-8", + ) + + rc = report_mod.main(["--table", str(f1), str(f2)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "spans battery fingerprints" in captured.err + assert "different task prompts/questions" in captured.err + + +def _report_main_markdown_flag(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") + rc = report_mod.main(["--table", "--markdown", str(f1)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.startswith("| task |") + assert "| R1 |" in out + assert "---" in out + + +def _report_main_no_dedupe_flag(tmp_path, capsys): + p = tmp_path / "d.jsonl" + rows = [ + _synth_row("R1", label="local", num_calls=1, success=True), + {**_synth_row("R1", label="local", num_calls=9, success=False)}, + ] + # Both rows have the same (task_id, rep, label), so latest-wins keeps one. + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + rc = report_mod.main(["--no-dedupe", str(p)]) + assert rc == 0 + # With no-dedupe, both rows enter summarize → n=2 for R1. + # (dedupe default would leave n=1.) + out = capsys.readouterr().out + assert "R1" in out + assert "2/2" in out or "1/2" in out # one success of two + + +@pytest.mark.parametrize( + "case", + case_params( + _report_marks_entirely_estimated_result_token_columns, + _report_marks_mixed_measured_and_estimated_columns, + _report_main_table_cli, + _report_main_table_warns_when_battery_fingerprints_differ, + _report_main_markdown_flag, + _report_main_no_dedupe_flag, + ), +) +def test_report_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) def test_format_surface_cell_variants(): @@ -194,81 +240,88 @@ def test_format_surface_cell_variants(): assert format_surface_cell(_synth_row("R1", skipped="nope")) == "skip" assert format_surface_cell(_synth_row("R1", error="boom")) == "ERR" assert format_surface_cell(_synth_row("R1", error_class="infra_seed", error="x")) == "ERR" - assert format_surface_cell(_synth_row("R1", success=True, num_calls=3, alt=0, oos=0)) == "✅ 3c" - assert format_surface_cell(_synth_row("R1", success=False, num_calls=4, alt=1, oos=1)) == "❌ 4c/2mp" - # external: no mispick suffix - assert format_surface_cell(_synth_row("R1", server="external", alt=None, oos=None, num_calls=5)) == "✅ 5c" - - -def test_multi_surface_behaviours(): - def test_multi_surface_table_snapshot_with_external(): - local = [ - _synth_row("R1", label="local", num_calls=4, alt=1, oos=0), - _synth_row("R2", label="local", success=False, num_calls=2), - ] - candidate = [ - _synth_row("R1", label="candidate", num_calls=2, alt=0, oos=0), - _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), - ] - external = [ - _synth_row("R1", label="akhil", server="external", alt=None, oos=None, num_calls=3), - _synth_row("R2", label="akhil", server="external", alt=None, oos=None, num_calls=1, success=False), - _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), - ] - table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) - assert table["columns"] == ["local", "candidate", "akhil"] - assert "R1" in table["task_ids"] and "R3" in table["task_ids"] - assert table["cells"]["R1"]["local"] == "✅ 4c/1mp" - assert table["cells"]["R1"]["candidate"] == "✅ 2c" - assert table["cells"]["R1"]["akhil"] == "✅ 3c" - assert table["cells"]["R2"]["candidate"] == "skip" - assert table["cells"]["R3"]["akhil"] == "ERR" - - text = render_multi_surface_table(table, markdown=False) - assert "local" in text and "candidate" in text and "akhil" in text - assert "✅ 3c" in text - assert "skip" in text - assert "ERR" in text - assert "infra 1" in text - - md = render_multi_surface_table(table, markdown=True) - assert md.startswith("| task |") - assert "| R1 |" in md - assert "---" in md - assert "**agg**" in md - - # Footer: external mispicks n/a - assert table["footer"]["akhil"]["mispicks"] is None - assert table["footer"]["local"]["mispicks"] == 1 - assert table["footer"]["akhil"]["infra_errors"] == 1 - - def test_multi_surface_table_aggregates_reps_and_flags_unstable(): - rows = [ - _synth_row("R1", rep=0, success=True, num_calls=2, label="local"), - _synth_row("R1", rep=1, success=True, num_calls=3, label="local"), - _synth_row("R1", rep=2, success=True, num_calls=2, label="local"), - _synth_row("R2", rep=0, success=True, num_calls=1, label="local"), - _synth_row("R2", rep=1, success=False, num_calls=4, label="local"), - _synth_row("R2", rep=2, success=True, num_calls=2, label="local"), - ] - - table = build_multi_surface_table([("local", rows)]) + assert format_surface_cell(_synth_row("R1", success=True, num_calls=3)) == "✅ 3c · tools —" + assert format_surface_cell(_synth_row("R1", success=False, num_calls=4)) == "❌ 4c · tools —" + assert format_surface_cell(_synth_row("R1", server="external", num_calls=5)) == "✅ 5c · tools —" + + +def _multi_surface_table_snapshot_with_external(): + local = [ + _synth_row("R1", label="local", num_calls=4), + _synth_row("R2", label="local", success=False, num_calls=2), + ] + candidate = [ + _synth_row("R1", label="candidate", num_calls=2), + _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), + ] + external = [ + _synth_row("R1", label="akhil", server="external", num_calls=3), + _synth_row("R2", label="akhil", server="external", num_calls=1, success=False), + _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), + ] + table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) + assert table["columns"] == ["local", "candidate", "akhil"] + assert "R1" in table["task_ids"] and "R3" in table["task_ids"] + assert table["cells"]["R1"]["local"] == "✅ 4c · tools —" + assert table["cells"]["R1"]["candidate"] == "✅ 2c · tools —" + assert table["cells"]["R1"]["akhil"] == "✅ 3c · tools —" + assert table["cells"]["R2"]["candidate"] == "skip" + assert table["cells"]["R3"]["akhil"] == "ERR" + + text = render_multi_surface_table(table, markdown=False) + assert "local" in text and "candidate" in text and "akhil" in text + assert "✅ 3c · tools —" in text + assert "skip" in text + assert "ERR" in text + assert "infra 1" in text + + md = render_multi_surface_table(table, markdown=True) + assert md.startswith("| task |") + assert "| R1 |" in md + assert "---" in md + assert "**agg**" in md + + assert table["footer"]["akhil"]["tool_variability"] is None + assert table["footer"]["local"]["tool_variability"] is None + assert table["footer"]["akhil"]["infra_errors"] == 1 + + +def _multi_surface_table_aggregates_reps_and_flags_unstable(): + rows = [ + _synth_row("R1", rep=0, success=True, num_calls=2, label="local", calls=[{"tool": "a"}, {"tool": "b"}]), + _synth_row("R1", rep=1, success=True, num_calls=3, label="local", calls=[{"tool": "a"}]), + _synth_row("R1", rep=2, success=True, num_calls=2, label="local", calls=[{"tool": "a"}]), + _synth_row("R2", rep=0, success=True, num_calls=1, label="local", calls=[{"tool": "c"}]), + _synth_row("R2", rep=1, success=False, num_calls=4, label="local", calls=[{"tool": "failed_only"}]), + _synth_row("R2", rep=2, success=True, num_calls=2, label="local", calls=[{"tool": "c"}]), + ] + + table = build_multi_surface_table([("local", rows)]) + + assert table["multi_rep"] is True + assert table["cells"]["R1"]["local"] == ( + "✅ 3/3 [0.44,1.00] 2-3c · tools success-only n=3; failed excluded=0; core:a(3c); variable:b=33%(1c)" + ) + assert table["cells"]["R2"]["local"] == ( + "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c · tools success-only n=2; failed excluded=1; core:c(2c)" + ) + assert table["footer"]["local"]["success"] == 5 + assert table["footer"]["local"]["n"] == 6 + assert table["footer"]["local"]["tool_variability"] == 1 + rendered = render_multi_surface_table(table) + assert "tool variability 1/2 tasks" in rendered + assert "noise floor" not in rendered - assert table["multi_rep"] is True - assert table["cells"]["R1"]["local"] == "✅ 3/3 [0.44,1.00] 2-3c" - assert table["cells"]["R2"]["local"] == "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c" - assert table["footer"]["local"]["success"] == 5 - assert table["footer"]["local"]["n"] == 6 - assert table["footer"]["local"]["unstable_tasks"] == 1 - rendered = render_multi_surface_table(table) - assert "measured noise floor: 1 task flipped at least once" in rendered - assert "minimum meaningful difference: 2 tasks" in rendered - test_multi_surface_table_snapshot_with_external() - test_multi_surface_table_aggregates_reps_and_flags_unstable() +@pytest.mark.parametrize( + "case", + case_params(_multi_surface_table_snapshot_with_external, _multi_surface_table_aggregates_reps_and_flags_unstable), +) +def test_multi_surface_behaviours(case): + case() -def test_single_rep_multi_surface_rendering_is_unchanged(): +def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): rows = [_synth_row("R1", label="local", success=True, num_calls=2)] rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) @@ -276,7 +329,9 @@ def test_single_rep_multi_surface_rendering_is_unchanged(): assert rendered == ( "task what local \n" "-------------------------------------------------------\n" - "R1 In project P, what is the curren… ✅ 2c\n" + "R1 In project P, what is the curren… ✅ 2c · tools —\n" "-------------------------------------------------------\n" - "local success 1/1 (100%) total calls 2 mispicks 0 infra 0\n" + "local success 1/1 (100%) total calls 2 tool variability — infra 0\n" + "local EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "local RUN COMPLETE: 1/1 rows completed\n" ) diff --git a/tests/evals/runner/test_canary.py b/tests/evals/runner/test_canary.py index db98e493..6e2b8204 100644 --- a/tests/evals/runner/test_canary.py +++ b/tests/evals/runner/test_canary.py @@ -1,109 +1,167 @@ -"""Offline eval tests for canary.""" +"""Offline eval tests for honest verifier-canary coverage.""" from __future__ import annotations import asyncio +import re +from types import SimpleNamespace +from typing import Any from unittest.mock import MagicMock -import pytest +from plane.errors.errors import HttpError from evals.runner import canary as runner_canary -from evals.runner import ( - run_canary, -) +from evals.runner import run_canary +from evals.tasks.schema import verify_s2 from evals.tasks.skip import TaskSkipped -def test_canary_behaviours(monkeypatch): - def test_canary_detects_broken_verifier(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - - async def always_ok(plane, ctx, run): - return True, "false positive" - - async def correctly_fails(plane, ctx, run): - return False, "empty agent correctly rejected" - - tasks = [ - { - "id": "GOOD", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": correctly_fails, - }, - { - "id": "BAD", - "prompt": "y {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": always_ok, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 - - def test_canary_passes_when_all_verifiers_reject(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - - async def reject(plane, ctx, run): - assert run == {"final_text": "", "calls": []} - return False, "no-op rejected" - - tasks = [ - { - "id": "G1", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": reject, - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 0 - - def test_canary_exits_1_when_all_tasks_skipped(monkeypatch): - fake_plane = MagicMock() - monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_canary, - "seed", - lambda *a, **k: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), +def _task(task_id: str, verify: Any) -> dict[str, Any]: + return { + "id": task_id, + "prompt": "x {project}", + "needs": set(), + "verify": verify, + } + + +def _install_harness(monkeypatch, *, plane=None, seed=None, teardown=None): + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (plane or MagicMock(), "test-ws")) + monkeypatch.setattr( + runner_canary, + "seed", + seed or (lambda *args, **kwargs: kwargs["ctx"].update({"project_name": "P", "project_id": "1"})), + ) + monkeypatch.setattr(runner_canary, "teardown", teardown or (lambda *args, **kwargs: None)) + + +async def _reject(_plane, _ctx, run): + assert run["calls"] == [] + assert run["call_source"] == "canary" + return False, "zero-call probe rejected" + + +def test_canary_detects_a_verifier_that_accepts_empty_output(monkeypatch): + _install_harness(monkeypatch) + + async def always_ok(_plane, _ctx, _run): + return True, "false positive" + + rc = asyncio.run(run_canary([_task("GOOD", _reject), _task("BAD", always_ok)], label="local")) + assert rc == 1 + + +def test_canary_accepts_a_fully_verified_set(monkeypatch, capsys): + _install_harness(monkeypatch) + rc = asyncio.run(run_canary([_task("G1", _reject)], label="local")) + assert rc == 0 + output = capsys.readouterr().out + assert "verified=1/1 ids=['G1']" in output + assert "skipped=0 ids=[]" in output + assert "errored=0 ids=[]" in output + + +def test_canary_treats_missing_s2_estimate_as_a_rejected_probe_not_an_error(monkeypatch, capsys): + def missing_estimate(**kwargs): + raise HttpError("Estimate not found", 404, {}) + + plane = SimpleNamespace(estimates=SimpleNamespace(retrieve=missing_estimate)) + + def seed(*args, **kwargs): + kwargs["ctx"].update({"workspace_slug": "test-ws", "project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, plane=plane, seed=seed) + rc = asyncio.run(run_canary([_task("S2", verify_s2)], label="local")) + + assert rc == 0 + output = capsys.readouterr().out + assert "verified=1/1 ids=['S2']" in output + assert "errored=0 ids=[]" in output + assert "requested Fibonacci scale was not created" in output + + +def test_canary_reports_partial_coverage_without_saying_all(monkeypatch, capsys): + def seed(*args, **kwargs): + if kwargs["task_id"] == "SKIP": + raise TaskSkipped("env:plan-gated:releases") + kwargs["ctx"].update({"project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run(run_canary([_task("GOOD", _reject), _task("SKIP", _reject)], label="local")) + assert rc == 0 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "verified=1/2 ids=['GOOD']" in output + assert "skipped=1 ids=['SKIP']" in output + assert "env:plan-gated:releases" in output + assert not re.search(r"\ball\b", output, flags=re.IGNORECASE) + + +def test_canary_strict_mode_fails_when_a_required_id_is_skipped(monkeypatch, capsys): + def seed(*args, **kwargs): + if kwargs["task_id"] == "SKIP": + raise TaskSkipped("env:plan-gated:releases") + kwargs["ctx"].update({"project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run( + run_canary( + [_task("GOOD", _reject), _task("SKIP", _reject)], + label="local", + required_task_ids={"GOOD", "SKIP"}, ) - monkeypatch.setattr(runner_canary, "teardown", lambda *a, **k: None) - tasks = [ - { - "id": "SKIPME", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (False, "unused"), - }, - ] - rc = asyncio.run(run_canary(tasks, label="local")) - assert rc == 1 - - with pytest.MonkeyPatch.context() as mp: - test_canary_detects_broken_verifier(mp) - with pytest.MonkeyPatch.context() as mp: - test_canary_passes_when_all_verifiers_reject(mp) - with pytest.MonkeyPatch.context() as mp: - test_canary_exits_1_when_all_tasks_skipped(mp) + ) + assert rc == 1 + assert "missing required ids=['SKIP']" in capsys.readouterr().err + + +def test_canary_teardown_error_affects_exit_and_error_report(monkeypatch, capsys): + def teardown(*args, **kwargs): + raise RuntimeError("cleanup failed") + + _install_harness(monkeypatch, teardown=teardown) + rc = asyncio.run(run_canary([_task("G1", _reject)], label="local")) + assert rc == 1 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "errored=1 ids=['G1']" in output + assert "teardown RuntimeError: cleanup failed" in output + + +def test_canary_labels_attachment_storage_seed_failure_as_infrastructure(monkeypatch, capsys): + def seed(*args, **kwargs): + raise ConnectionError("localhost:9000 attachment storage unreachable") + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run(run_canary([_task("L5", _reject)], label="local")) + + assert rc == 1 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "L5 canary ERROR[infra_seed]" in output + assert "infra_seed ConnectionError: localhost:9000 attachment storage unreachable" in output + assert "skipped=0 ids=[]" in output + + +def test_canary_catches_adversarial_canned_contract_output(monkeypatch, capsys): + _install_harness(monkeypatch) + + async def accepts_fabricated_count(_plane, _ctx, run): + if run["final_text"] == "": + return False, "empty rejected" + return run["final_text"] == "count: 0", "fabricated zero accepted" + + rc = asyncio.run(run_canary([_task("R2", accepts_fabricated_count)], label="local")) + assert rc == 1 + output = capsys.readouterr().out + assert "accepted canary probe" in output + assert "count: 0" in output + + +def test_canary_exits_nonzero_when_no_task_is_verified(monkeypatch): + _install_harness( + monkeypatch, + seed=lambda *args, **kwargs: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), + ) + rc = asyncio.run(run_canary([_task("SKIPME", _reject)], label="local")) + assert rc == 1 diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 8efe41ba..2c34782b 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -6,6 +6,7 @@ import json import subprocess from pathlib import Path +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock @@ -16,6 +17,7 @@ from evals.drivers import ( ClaudeCliDriver, ) +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.report import load_rows, summarize from evals.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult from evals.runner import ( @@ -25,7 +27,7 @@ from evals.runner import live as runner_live from evals.runner.live import stdio_server_env from evals.tasks.skip import TaskSkipped -from tests.evals.conftest import _data_rows +from tests.evals.conftest import _data_rows, case_params def _taxonomy_task( @@ -39,37 +41,37 @@ def _taxonomy_task( "id": task_id, "prompt": prompt, "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": {"search_work_items"}, - "optimal_calls": 1, "needs": set(needs or set()), "verify": verify, } -def test_stdio_behaviours(monkeypatch): - def test_stdio_env_still_works_for_cli_drivers(monkeypatch): - monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") - monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") - monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) - env = stdio_server_env() - assert env["PLANE_API_KEY"] == "k" - assert "ANTHROPIC_API_KEY" not in env +def _stdio_env_still_works_for_cli_drivers(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + env = stdio_server_env() + assert env["PLANE_API_KEY"] == "k" + assert "ANTHROPIC_API_KEY" not in env - def test_stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): - monkeypatch.setenv("SOME_SECRET", "x") - environment = runner_live.stdio_server_env() +def _stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): + monkeypatch.setenv("SOME_SECRET", "x") - assert "SOME_SECRET" not in environment - assert environment["PLANE_API_KEY"] == "test-key" - assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" - assert environment["PLANE_BASE_URL"] == "https://api.plane.so" + environment = runner_live.stdio_server_env() - with pytest.MonkeyPatch.context() as mp: - test_stdio_env_still_works_for_cli_drivers(mp) - with pytest.MonkeyPatch.context() as mp: - test_stdio_server_env_does_not_leak_ambient_secrets(mp) + assert "SOME_SECRET" not in environment + assert environment["PLANE_API_KEY"] == "test-key" + assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" + assert environment["PLANE_BASE_URL"] == "https://api.plane.so" + + +@pytest.mark.parametrize( + "case", + case_params(_stdio_env_still_works_for_cli_drivers, _stdio_server_env_does_not_leak_ambient_secrets), +) +def test_stdio_behaviours(monkeypatch, case): + case(monkeypatch) def test_live_run_rejects_non_positive_reps(capsys): @@ -77,766 +79,758 @@ def test_live_run_rejects_non_positive_reps(capsys): assert "--reps must be at least 1" in capsys.readouterr().err -def test_run_behaviours(tmp_path, monkeypatch, capsys): - def test_run_live_seed_failure_is_infra_seed(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - - fake_plane = MagicMock() - driver = MagicMock() - torn: list[dict[str, Any]] = [] - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - - def boom_seed(plane, run_id, needs, ctx): - ctx["project_name"] = "EVAL deadbeef" - raise HttpError("identifier already taken", 409) - - monkeypatch.setattr(runner_live, "seed", boom_seed) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - task = { - "id": "T1", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - } +def _run_seed_failure_is_infra_seed(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" - rc = asyncio.run( - run_live( - [task], - model_alias="standard", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - resolved_model_id="sonnet", - ) - ) - assert rc == 0 - rows = _data_rows(out) - assert len(rows) == 1 - row = rows[0] - assert row["schema_version"] == RESULT_SCHEMA_VERSION - assert row["error_class"] == "infra_seed" - assert row["success"] is False - assert row["verify_note"] == "" - assert "HttpError" in (row["error"] or "") - assert "identifier" in (row["error"] or "").lower() - assert row["battery"] # fingerprint written - assert row["requested_model"] == "standard" - assert row["requested_tier"] == "standard" - assert row["resolved_model"] == "sonnet" - assert row["model"] == "sonnet" - meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) - assert meta["schema_version"] == RESULT_SCHEMA_VERSION - assert meta["requested_tier"] == "standard" - assert meta["resolved_model"] == "sonnet" - driver.run_task.assert_not_called() - assert torn == [{"project_name": "EVAL deadbeef"}] - - def test_run_live_missing_bug_type_uses_context_skip_reason(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - torn: list[dict[str, Any]] = [] - - def seed_without_bug_type(plane, run_id, needs, ctx): - ctx.update( - { - "project_name": "EVAL no bug type", - "project_id": "p1", - "bug_type_skip_reason": "plan:work-item-types-disabled", - } - ) + fake_plane = MagicMock() + driver = MagicMock() + torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + + def boom_seed(plane, run_id, needs, ctx, task_id=None): + ctx["project_name"] = "EVAL deadbeef" + raise HttpError("identifier already taken", 409) - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + monkeypatch.setattr(runner_live, "seed", boom_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - task = _taxonomy_task( - "BUGTYPE", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - needs={"bug_type"}, + task = { + "id": "T1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + } + + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + resolved_model_id="sonnet", ) - rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) - - assert rc == 0 - row = _data_rows(out)[0] - assert row["skipped"] == "plan:work-item-types-disabled" - assert row["verify_note"] == "plan:work-item-types-disabled" - assert row["error"] is None - assert row["error_class"] is None - driver.run_task.assert_not_called() - assert torn == [ + ) + assert rc == 1 + rows = _data_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["error_class"] == "infra_seed" + assert row["success"] is False + assert row["verify_note"] == "" + assert "HttpError" in (row["error"] or "") + assert "identifier" in (row["error"] or "").lower() + assert row["battery"] # fingerprint written + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "sonnet" + assert row["model"] == "sonnet" + meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert meta["schema_version"] == RESULT_SCHEMA_VERSION + assert meta["requested_tier"] == "standard" + assert meta["resolved_model"] == "sonnet" + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL deadbeef"}] + + +def _run_missing_bug_type_uses_context_skip_reason(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + def seed_without_bug_type(plane, run_id, needs, ctx, task_id=None): + ctx.update( { "project_name": "EVAL no bug type", "project_id": "p1", - "bug_type_skip_reason": "plan:work-item-types-disabled", + "bug_type_skip_reason": "env:plan-gated:work-item-types", } - ] - - def test_run_live_prompt_bind_failure_is_infra_seed(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - torn: list[dict[str, Any]] = [] - - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - task = _taxonomy_task( - "PROMPT", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - prompt="use {missing_seed_id} in {project}", - ) - rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) - - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_seed" - assert row["verify_note"] == "" - assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") - driver.run_task.assert_not_called() - assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] - - def test_run_live_api_driver_exception_is_infra_api(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.side_effect = RuntimeError("provider unavailable") - torn: list[dict[str, Any]] = [] - - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - task = _taxonomy_task( - "APIERR", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), - ) - rc = asyncio.run( - run_live( - [task], - model_alias="standard", - reps=1, - label="local", - out_path=out, - driver_name="api", - resolved_model_id="provider-model-id", - ) - ) + task = _taxonomy_task( + "BUGTYPE", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + needs={"bug_type"}, + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_api" - assert row["verify_note"] == "" - assert row["success"] is False - assert row["error"] == "RuntimeError: provider unavailable" - driver.run_task.assert_called_once() - assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] - - def test_run_live_driver_exception_is_infra_cli(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - - def ok_seed(plane, run_id, needs, ctx): - ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) - - monkeypatch.setattr(runner_live, "seed", ok_seed) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - - class BoomDriver: - name = "claude-cli" - - def run_task(self, *args, **kwargs): - raise RuntimeError("claude cli failed: json_parse_failed") - - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) - - task = { - "id": "T2", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": lambda *a, **k: (False, "nope"), + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "env:plan-gated:work-item-types" + assert row["verify_note"] == "env:plan-gated:work-item-types" + assert row["error"] is None + assert row["error_class"] is None + driver.run_task.assert_not_called() + assert torn == [ + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "env:plan-gated:work-item-types", } + ] - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert "RuntimeError" in (row["error"] or "") - - def test_run_live_timeout_agent_is_infra_cli(tmp_path, monkeypatch): - from evals.results import agent_run_to_harness_dict - - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - class TimeoutDriver: - name = "claude-cli" +def _run_prompt_bind_failure_is_infra_seed(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] - def run_task(self, *args, **kwargs): - return AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="timeout", - notes=["timeout after 900s"], - ) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) + task = _taxonomy_task( + "PROMPT", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + prompt="use {missing_seed_id} in {project}", + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) - verify_calls: list[Any] = [] + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["verify_note"] == "" + assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] - async def verify(*a, **k): - verify_calls.append(1) - return True, "should not run" - task = { - "id": "T3", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } +def _run_api_driver_exception_is_infra_api(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.side_effect = RuntimeError("provider unavailable") + torn: list[dict[str, Any]] = [] - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed - assert row["stop_reason"] == "timeout" - assert verify_calls == [] - d = agent_run_to_harness_dict( - AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout"), - optimal=set(), - alternate=set(), - classify=lambda t, o, a: "out_of_set", - ) - assert d["stop_reason"] == "timeout" - - def test_run_live_error_during_execution_is_infra_cli(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "APIERR", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + ) + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="api", + resolved_model_id="provider-model-id", ) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - payload = { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": "MCP server crashed", - "session_id": "sess-err", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } + ) - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_api" + assert row["verify_note"] == "" + assert row["success"] is False + assert row["error"] == "RuntimeError: provider unavailable" + driver.run_task.assert_called_once() + assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) - verify_calls: list[Any] = [] +def _run_driver_exception_is_infra_cli(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - async def verify(*a, **k): - verify_calls.append(1) - return False, "nope" + def ok_seed(plane, run_id, needs, ctx, task_id=None): + ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) - task = { - "id": "T4", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] == "infra_cli" - assert row["stop_reason"] == "error_during_execution" - assert verify_calls == [] - assert "claude_exit=1" in (row.get("driver_notes") or []) - - def test_run_live_error_max_turns_is_task_path(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - monkeypatch.setattr( - runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"}) - ) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - payload = { - "type": "result", - "subtype": "error_max_turns", - "is_error": True, - "result": "hit max turns", - "session_id": "sess-max", - "num_turns": 15, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") + class BoomDriver: + name = "claude-cli" - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + def run_task(self, *args, **kwargs): + raise RuntimeError("claude cli failed: json_parse_failed") - verify_calls: list[Any] = [] + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) - async def verify(*a, **k): - verify_calls.append(1) - return False, "agent exhausted turns" + task = { + "id": "T2", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": lambda *a, **k: (False, "nope"), + } - task = { - "id": "T5", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify, - } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=out, - driver_name="claude-cli", - ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["error_class"] is None - assert row["stop_reason"] == "error_max_turns" - assert row["success"] is False - assert verify_calls == [1] - - def test_run_live_verifier_skip_is_not_a_failure(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[], - final_text="done", - usage=None, - stopped_reason="end_turn", + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", ) - torn: list[dict[str, Any]] = [] + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert "RuntimeError" in (row["error"] or "") - async def skip_verify(plane, ctx, run): - raise TaskSkipped("env:verification-unavailable") - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), - ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("VERIFYSKIP", skip_verify)], - model_alias="standard", - reps=1, - label="local", - out_path=out, +def _run_timeout_agent_is_infra_cli(tmp_path, monkeypatch, _capsys): + from evals.results import agent_run_to_harness_dict + + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + class TimeoutDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="timeout", + notes=["timeout after 900s"], ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["skipped"] == "env:verification-unavailable" - assert row["verify_note"] == "env:verification-unavailable" - assert row["success"] is False - assert row["error"] is None - assert row["error_class"] is None - assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] - - def test_run_live_verifier_exception_is_task_error(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[], - final_text="done", - usage=None, - stopped_reason="end_turn", + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return True, "should not run" + + task = { + "id": "T3", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", ) - torn: list[dict[str, Any]] = [] + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed + assert row["stop_reason"] == "timeout" + assert verify_calls == [] + d = agent_run_to_harness_dict(AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout")) + assert d["stop_reason"] == "timeout" + + +def _run_error_during_execution_is_infra_cli(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "MCP server crashed", + "session_id": "sess-err", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + + verify_calls: list[Any] = [] - async def broken_verify(plane, ctx, run): - raise ValueError("verifier broke") + async def verify(*a, **k): + verify_calls.append(1) + return False, "nope" - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), + task = { + "id": "T4", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("VERIFYERR", broken_verify)], - model_alias="standard", - reps=1, - label="local", - out_path=out, - ) + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["stop_reason"] == "error_during_execution" + assert verify_calls == [] + assert "claude_exit=1" in (row.get("driver_notes") or []) + + +def _run_error_max_turns_is_task_path(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_max_turns", + "is_error": True, + "result": "hit max turns", + "session_id": "sess-max", + "num_turns": 15, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return False, "agent exhausted turns" + + task = { + "id": "T5", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] is None + assert row["stop_reason"] == "error_max_turns" + assert row["success"] is False + assert verify_calls == [1] + + +def _run_verifier_skip_is_not_a_failure(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def skip_verify(plane, ctx, run): + raise TaskSkipped("env:verification-unavailable") - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is False - assert row["error_class"] == "task" - assert row["error"] == "ValueError: verifier broke" - assert row["verify_note"] == "" - assert row["skipped"] is None - assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] - - def test_run_live_external_server_nulls_catalog_mispicks(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[{"tool": "search_work_items", "args": {}}], - final_text="done", - usage=None, - stopped_reason="end_turn", + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYSKIP", skip_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["skipped"] == "env:verification-unavailable" + assert row["verify_note"] == "env:verification-unavailable" + assert row["success"] is False + assert row["error"] is None + assert row["error_class"] is None + assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] - async def verify_ok(plane, ctx, run): - return True, "external ok" - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), +def _run_verifier_exception_is_task_error(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def broken_verify(plane, ctx, run): + raise ValueError("verifier broke") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYERR", broken_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("EXTERNAL", verify_ok)], - model_alias="standard", - reps=1, - label="local", - out_path=out, - server_cmd=["/bin/foreign", "stdio"], - ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["success"] is False + assert row["error_class"] == "task" + assert row["error"] == "ValueError: verifier broke" + assert row["verify_note"] == "" + assert row["skipped"] is None + assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] + + +def _run_external_server_records_observed_calls(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "search_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "external ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("EXTERNAL", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + server_cmd=["/bin/foreign", "stdio"], ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["server"] == "external" + assert row["num_calls"] == 1 + assert row["calls"][0]["tool"] == "search_work_items" - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is True - assert row["server"] == "external" - assert row["alternate_calls"] is None - assert row["out_of_set_calls"] is None - - def test_run_live_success_keeps_requested_and_resolved_models(tmp_path, monkeypatch): - out = tmp_path / "rows.jsonl" - driver = MagicMock() - driver.run_task.return_value = AgentRun( - calls=[{"tool": "list_work_items", "args": {}}], - final_text="done", - usage=None, - stopped_reason="end_turn", + +def _run_success_keeps_requested_and_resolved_models(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "list_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "local ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("SUCCESS", verify_ok)], + model_alias="standard", + resolved_model_id="provider-model-id", + reps=1, + label="local", + out_path=out, ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "provider-model-id" + assert row["server"] == "local" + + +def _run_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path, monkeypatch, _capsys): + out = tmp_path / "multi.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_ids: list[str] = [] + teardown_projects: list[str] = [] + + def fresh_seed(plane, run_id, needs, ctx, task_id=None): + seed_ids.append(run_id) + ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + + def record_teardown(plane, ctx): + teardown_projects.append(ctx["project_id"]) - async def verify_ok(plane, ctx, run): - return True, "local ok" + async def fake_agent(**kwargs): + return TaskResult(final_text="done", stop_reason="end_turn") - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) - monkeypatch.setattr( - runner_live, - "seed", - lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), + monkeypatch.setattr(runner_live, "seed", fresh_seed) + monkeypatch.setattr(runner_live, "teardown", record_teardown) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) + monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + task = { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=3, + label="local", + out_path=out, + driver_name="claude-cli", ) - monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) - - rc = asyncio.run( - run_live( - [_taxonomy_task("SUCCESS", verify_ok)], - model_alias="standard", - resolved_model_id="provider-model-id", - reps=1, - label="local", - out_path=out, + ) + + assert rc == 0 + assert len(seed_ids) == 3 + assert len(set(seed_ids)) == 3 + assert teardown_projects == seed_ids + rows = _data_rows(out) + assert [row["rep"] for row in rows] == [0, 1, 2] + assert all(row["success"] is True for row in rows) + + +def test_runner_passes_seeded_evidence_to_driver_and_retains_only_labels(monkeypatch): + sentinel = "hidden-target-fact-2f81a0cd" + captured: dict[str, Any] = {} + + class Driver: + def run_task(self, *args, **kwargs): + captured.update(kwargs) + return AgentRun( + calls=[ + { + "tool": "read_any_route", + "args": {}, + "is_error": False, + "result_chars": 42, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + final_text="count: 4", + usage=None, + stopped_reason="end_turn", + call_source="api", + evidence_trace_available=True, ) - ) - assert rc == 0 - row = _data_rows(out)[0] - assert row["success"] is True - assert row["requested_model"] == "standard" - assert row["requested_tier"] == "standard" - assert row["resolved_model"] == "provider-model-id" - assert row["server"] == "local" - - def test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path, monkeypatch): - out = tmp_path / "multi.jsonl" - fake_plane = MagicMock() - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) - seed_ids: list[str] = [] - teardown_projects: list[str] = [] - - def fresh_seed(plane, run_id, needs, ctx): - seed_ids.append(run_id) - ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) - - def record_teardown(plane, ctx): - teardown_projects.append(ctx["project_id"]) - - async def fake_agent(**kwargs): - return TaskResult(final_text="done", stop_reason="end_turn") - - monkeypatch.setattr(runner_live, "seed", fresh_seed) - monkeypatch.setattr(runner_live, "teardown", record_teardown) - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) - monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) - - async def verify_ok(plane, ctx, run): - return True, "ok" - - task = { - "id": "R1", - "prompt": "do {project}", - "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": verify_ok, - } + monkeypatch.setenv("EVAL_PLANE_API_KEY", "key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + task = {"id": "R2", "prompt": "In {project}, count.", "tags": {"read"}, "needs": {"items"}} + context = { + "project_name": "EVAL deadbeef", + "evidence_sentinels": {TARGET_ENTITY_EVIDENCE: [sentinel]}, + } - rc = asyncio.run( - run_live( - [task], - model_alias="sonnet", - reps=3, - label="local", - out_path=out, - driver_name="claude-cli", - ) + row = asyncio.run( + runner_live.run_agent_task_via_driver( + driver=Driver(), + model_id="model", + task=task, + ctx=context, + workspace_slug="ws", ) + ) + + assert captured["evidence_sentinels"] == context["evidence_sentinels"] + assert row.evidence_trace_available is True + assert row.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] + assert sentinel not in json.dumps(row.to_row()) - assert rc == 0 - assert len(seed_ids) == 3 - assert len(set(seed_ids)) == 3 - assert teardown_projects == seed_ids - rows = _data_rows(out) - assert [row["rep"] for row in rows] == [0, 1, 2] - assert all(row["success"] is True for row in rows) - def test_run_live_passes_server_cmd_to_non_claude(monkeypatch, tmp_path): - from evals.runner import live as run_mod +def _run_passes_server_cmd_to_non_claude(tmp_path, monkeypatch, _capsys): + from evals.runner import live as run_mod - captured: dict = {} + captured: dict = {} - def fake_get_driver(name, **kwargs): - captured["name"] = name - captured["kwargs"] = kwargs + def fake_get_driver(name, **kwargs): + captured["name"] = name + captured["kwargs"] = kwargs - class Dummy: - def run_task(self, *a, **k): - return AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - call_source="json", - ) + class Dummy: + def run_task(self, *a, **k): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + call_source="json", + ) - return Dummy() + return Dummy() - monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) - monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) - monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) - import asyncio + import asyncio - async def _verify(*a, **k): - return False, "n" + async def _verify(*a, **k): + return False, "n" - task = { - "id": "T", - "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, - "needs": set(), - "verify": _verify, - } - rc = asyncio.run( - run_mod.run_live( - [task], - model_alias="sonnet", - reps=1, - label="local", - out_path=tmp_path / "o.jsonl", - driver_name="opencode-cli", - server_cmd=["/bin/foreign", "stdio"], - ) + task = { + "id": "T", + "prompt": "x {project}", + "needs": set(), + "verify": _verify, + } + rc = asyncio.run( + run_mod.run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=tmp_path / "o.jsonl", + driver_name="opencode-cli", + server_cmd=["/bin/foreign", "stdio"], ) - assert rc == 0 - assert captured["name"] == "opencode-cli" - assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] - - def test_run_live_reports_progress_per_repetition(tmp_path, monkeypatch, capsys): - out = tmp_path / "out.jsonl" - - async def passes(_plane, _ctx, _run): - return True, "ok" - - monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) - monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) - monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) - - async def fake_drive(**kwargs): - return TaskResult(final_text="done", num_calls=2) - - monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) - monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) - - tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] - rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) - assert rc == 0 - - printed = capsys.readouterr().out - # Position out of total, before the task runs. - assert "[ 1/2] R1 rep=0 running" in printed - assert "[ 2/2] R2 rep=0 running" in printed - # A running tally after each, and one closing summary. - assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed - assert "finished 2/2 in " in printed - assert "2 pass, 0 fail, 0 skip" in printed - - _d0 = tmp_path / "test_run_live_seed_failure_is_infra_seed" - _d0.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_seed_failure_is_infra_seed(_d0, mp) - _d1 = tmp_path / "test_run_live_missing_bug_type_uses_context_skip_reason" - _d1.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_missing_bug_type_uses_context_skip_reason(_d1, mp) - _d2 = tmp_path / "test_run_live_prompt_bind_failure_is_infra_seed" - _d2.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_prompt_bind_failure_is_infra_seed(_d2, mp) - _d3 = tmp_path / "test_run_live_api_driver_exception_is_infra_api" - _d3.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_api_driver_exception_is_infra_api(_d3, mp) - _d4 = tmp_path / "test_run_live_driver_exception_is_infra_cli" - _d4.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_driver_exception_is_infra_cli(_d4, mp) - _d5 = tmp_path / "test_run_live_timeout_agent_is_infra_cli" - _d5.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_timeout_agent_is_infra_cli(_d5, mp) - _d6 = tmp_path / "test_run_live_error_during_execution_is_infra_cli" - _d6.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_error_during_execution_is_infra_cli(_d6, mp) - _d7 = tmp_path / "test_run_live_error_max_turns_is_task_path" - _d7.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_error_max_turns_is_task_path(_d7, mp) - _d8 = tmp_path / "test_run_live_verifier_skip_is_not_a_failure" - _d8.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_verifier_skip_is_not_a_failure(_d8, mp) - _d9 = tmp_path / "test_run_live_verifier_exception_is_task_error" - _d9.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_verifier_exception_is_task_error(_d9, mp) - _d10 = tmp_path / "test_run_live_external_server_nulls_catalog_mispicks" - _d10.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_external_server_nulls_catalog_mispicks(_d10, mp) - _d11 = tmp_path / "test_run_live_success_keeps_requested_and_resolved_models" - _d11.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_success_keeps_requested_and_resolved_models(_d11, mp) - _d12 = tmp_path / "test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep" - _d12.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_multi_rep_uses_fresh_seed_and_teardown_per_rep(_d12, mp) - _d13 = tmp_path / "test_run_live_passes_server_cmd_to_non_claude" - _d13.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_passes_server_cmd_to_non_claude(mp, _d13) - _d14 = tmp_path / "test_run_live_reports_progress_per_repetition" - _d14.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_live_reports_progress_per_repetition(_d14, mp, capsys) + ) + assert rc == 0 + assert captured["name"] == "opencode-cli" + assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] + + +def _run_reports_progress_per_repetition(tmp_path, monkeypatch, capsys): + out = tmp_path / "out.jsonl" + + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + return TaskResult(final_text="done", num_calls=2) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + assert rc == 0 + + printed = capsys.readouterr().out + # Position out of total, before the task runs. + assert "[ 1/2] R1 rep=0 running" in printed + assert "[ 2/2] R2 rep=0 running" in printed + # A running tally after each, and one closing summary. + assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed + assert "finished 2/2 in " in printed + assert "2 pass, 0 fail, 0 skip" in printed + + +_RUN_CASES = case_params( + _run_seed_failure_is_infra_seed, + _run_missing_bug_type_uses_context_skip_reason, + _run_prompt_bind_failure_is_infra_seed, + _run_api_driver_exception_is_infra_api, + _run_driver_exception_is_infra_cli, + _run_timeout_agent_is_infra_cli, + _run_error_during_execution_is_infra_cli, + _run_error_max_turns_is_task_path, + _run_verifier_skip_is_not_a_failure, + _run_verifier_exception_is_task_error, + _run_external_server_records_observed_calls, + _run_success_keeps_requested_and_resolved_models, + _run_multi_rep_uses_fresh_seed_and_teardown_per_rep, + _run_passes_server_cmd_to_non_claude, + _run_reports_progress_per_repetition, +) + + +@pytest.mark.parametrize("case", _RUN_CASES) +def test_run_behaviours(case, tmp_path, monkeypatch, capsys): + case(tmp_path, monkeypatch, capsys) def test_is_infra_cli_stop_reason_matrix(): @@ -876,9 +870,6 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: { "id": "L2", "prompt": "x {project}", - "optimal_tools": {"a"}, - "alternate_tools": set(), - "optimal_calls": 1, "needs": {"activity_feed"}, "verify": None, # never reached }, @@ -901,6 +892,190 @@ def skip_seed(*_args: Any, **_kwargs: Any) -> None: summary = summarize(load_rows(out)) assert "L2" not in summary.tasks assert summary.aggregate_n == 0 + assert summary.expected_skips == 1 + assert summary.unexpected_skips == 0 + assert summary.complete is True + + +def test_attachment_storage_connection_failure_is_infra_seed(tmp_path, monkeypatch): + out = tmp_path / "out.jsonl" + driver = MagicMock() + + def storage_unreachable(*_args, **_kwargs): + raise ConnectionError("localhost:9000 attachment storage unreachable") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", storage_unreachable) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: driver) + + rc = asyncio.run( + run_live([_taxonomy_task("L5", None)], model_alias="standard", reps=1, label="local", out_path=out) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["task_id"] == "L5" + assert row["error_class"] == "infra_seed" + assert row["error"] == "ConnectionError: localhost:9000 attachment storage unreachable" + assert row["skipped"] is None + driver.run_task.assert_not_called() + + +def test_activity_read_connection_failure_is_infra_seed_and_incomplete(tmp_path, monkeypatch, capsys): + from evals.seed.work_items import CHECKOUT_TIMEOUT_TITLE, require_activities + + out = tmp_path / "out.jsonl" + driver = MagicMock() + + def activity_backend_unreachable(**kwargs): + raise ConnectionError("activity backend unreachable") + + plane = SimpleNamespace(work_items=SimpleNamespace(activities=SimpleNamespace(list=activity_backend_unreachable))) + + def seed_l2(plane, run_id, needs, ctx, task_id=None): + ctx.update( + { + "workspace_slug": "ws", + "project_id": "project-1", + "items": {CHECKOUT_TIMEOUT_TITLE: "work-item-1"}, + } + ) + require_activities(plane, "ws", ctx) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (plane, "ws")) + monkeypatch.setattr(runner_live, "seed", seed_l2) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("L2", None, needs={"activity_feed"})], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["error"] == "ConnectionError: activity backend unreachable" + assert row["skipped"] is None + assert "RUN INCOMPLETE:" in capsys.readouterr().out + driver.run_task.assert_not_called() + + +@pytest.mark.parametrize( + ("reason", "expected_rc", "verdict"), + [ + ("env:plan-gated:customers", 0, "RUN COMPLETE:"), + ("env:no-activity-worker", 0, "RUN COMPLETE:"), + ("env:plan-gated:customerz", 1, "RUN INCOMPLETE:"), + ("env:fixture-collision:customers:Acme Corp", 1, "RUN INCOMPLETE:"), + ("env:new-skip-reason", 1, "RUN INCOMPLETE:"), + ], +) +def test_run_live_completeness_skip_taxonomy(tmp_path, monkeypatch, capsys, reason, expected_rc, verdict): + out = tmp_path / "out.jsonl" + + def skip_seed(*_args, **_kwargs): + raise TaskSkipped(reason) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", skip_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + tasks = [_taxonomy_task("R1", None), _taxonomy_task("R2", None)] + + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == expected_rc + output = capsys.readouterr().out + assert "success: 0/0" in output + assert "EXECUTION COVERAGE: 0/2 rows evaluated (0.0%)" in output + assert f"R1,R2 ({reason})" in output + assert verdict in output + if reason.startswith("env:fixture-collision:"): + assert "unexpected skips=2 [fixture-collision=2]" in output + + +def test_run_live_cleanup_failure_is_incomplete_without_changing_success(tmp_path, monkeypatch, capsys): + out = tmp_path / "out.jsonl" + delete_calls: list[tuple[str, str]] = [] + + def fail_delete(kind: str, object_id: str) -> None: + delete_calls.append((kind, object_id)) + raise RuntimeError(f"delete failed for {kind} {object_id}") + + class _Page: + results: list[Any] = [] + next_page_results = False + next_cursor = None + + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page(), + delete=lambda **kw: fail_delete("customer", kw["customer_id"]), + properties=SimpleNamespace(list=lambda **kw: _Page(), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page(), + delete=lambda **kw: fail_delete("release_tag", kw["tag_id"]), + ) + ), + ) + driver = MagicMock() + driver.run_task.return_value = AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(*_args, **_kwargs): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (plane, "ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update( + { + "workspace_slug": "ws", + "project_name": "EVAL cleanup", + "project_id": None, + "workspace_objects": [ + {"kind": "customer", "id": "customer-1"}, + {"kind": "release_tag", "id": "tag-1"}, + ], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + ), + ) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("R1", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["cleanup_error"].startswith("TeardownError: 2 cleanup operation(s) failed:") + assert delete_calls == [("customer", "customer-1"), ("release_tag", "tag-1")] + output = capsys.readouterr().out + assert "success: 1/1 (100.0%)" in output + assert "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)" in output + assert "RUN INCOMPLETE:" in output + assert "cleanup errors=1" in output # --------------------------------------------------------------------------- diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py index 0493a3eb..9d8de88f 100644 --- a/tests/evals/runner/test_resume.py +++ b/tests/evals/runner/test_resume.py @@ -22,206 +22,229 @@ should_skip_resume_row, ) from evals.runner import live as runner_live -from tests.evals.conftest import _data_rows - - -def test_should_skip_behaviours(): - def test_should_skip_resume_row_completed_success(): - assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True - - def test_should_skip_resume_row_verify_fail_without_error(): - assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True - - def test_should_skip_resume_row_infra_seed_retries(): - assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False - - def test_should_skip_resume_row_infra_cli_retries(): - assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False - - def test_should_skip_resume_row_non_null_error_retries(): - assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False - assert should_skip_resume_row({"error": "boom", "error_class": None}) is False - - test_should_skip_resume_row_completed_success() - test_should_skip_resume_row_verify_fail_without_error() - test_should_skip_resume_row_infra_seed_retries() - test_should_skip_resume_row_infra_cli_retries() - test_should_skip_resume_row_non_null_error_retries() - - -def test_load_behaviours(tmp_path, capsys): - def test_load_resume_skip_keys_summary(tmp_path): - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, - {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, - {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local"), ("W1", 0, "local")} - assert n_skip == 2 - assert n_retry == 1 - - def test_load_resume_skip_keys_n_retry_ignores_later_success(tmp_path): - p = tmp_path / "out.jsonl" - rows = [ - {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, - {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - assert n_retry == 0 - - def test_load_resume_skip_keys_label_mismatch(tmp_path): - p = tmp_path / "out.jsonl" - p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") - with pytest.raises(SystemExit, match="label"): - load_resume_skip_keys(p, label="local") - - def test_load_resume_skip_keys_battery_model_driver_mismatch(tmp_path): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "battery": "aaaaaaaaaaaa", - "model": "sonnet", - "driver": "claude-cli", - "error": None, - } - ) - + "\n", - encoding="utf-8", - ) - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") - with pytest.raises(SystemExit, match="driver"): - load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") - # Missing keys on older rows: pass (back-compat) - p2 = tmp_path / "old.jsonl" - p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") - skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") - assert ("R1", 0, "local") in skip - - def test_load_resume_skip_keys_truncated_json(tmp_path, capsys): - p = tmp_path / "out.jsonl" - p.write_text( - json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) - + "\n" - + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated - encoding="utf-8", +from tests.evals.conftest import _data_rows, case_params + + +def _should_skip_resume_row_completed_success(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True + + +def _should_skip_resume_row_verify_fail_without_error(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True + + +def _should_skip_resume_row_infra_seed_retries(): + assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False + + +def _should_skip_resume_row_infra_cli_retries(): + assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False + + +def _should_skip_resume_row_non_null_error_retries(): + assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False + assert should_skip_resume_row({"error": "boom", "error_class": None}) is False + + +def _should_skip_resume_row_only_plan_gated_skip_is_terminal(): + assert should_skip_resume_row({"skipped": "env:plan-gated:customers"}) is True + assert should_skip_resume_row({"skipped": "env:no-activity-worker"}) is True + assert should_skip_resume_row({"skipped": "env:no-activity-worker (worker disabled)"}) is False + assert should_skip_resume_row({"skipped": "env:plan-gated:customerz"}) is False + assert should_skip_resume_row({"skipped": "env:fixture-collision:customers:Acme Corp"}) is False + assert should_skip_resume_row({"skipped": "env:unknown"}) is False + assert should_skip_resume_row({"cleanup_error": "TeardownError: delete failed"}) is False + + +@pytest.mark.parametrize( + "case", + case_params( + _should_skip_resume_row_completed_success, + _should_skip_resume_row_verify_fail_without_error, + _should_skip_resume_row_infra_seed_retries, + _should_skip_resume_row_infra_cli_retries, + _should_skip_resume_row_non_null_error_retries, + _should_skip_resume_row_only_plan_gated_skip_is_terminal, + ), +) +def test_should_skip_behaviours(case): + case() + + +def _load_resume_skip_keys_summary(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local"), ("W1", 0, "local")} + assert n_skip == 2 + assert n_retry == 1 + + +def _load_resume_skip_keys_n_retry_ignores_later_success(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + assert n_retry == 0 + + +def _load_resume_skip_keys_label_mismatch(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") + with pytest.raises(SystemExit, match="label"): + load_resume_skip_keys(p, label="local") + + +def _load_resume_skip_keys_battery_model_driver_mismatch(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "aaaaaaaaaaaa", + "model": "sonnet", + "driver": "claude-cli", + "error": None, + } ) - skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") - assert skip == {("R1", 0, "local")} - assert n_skip == 1 - err = capsys.readouterr().err - assert "invalid JSON" in err - - def test_load_resume_skip_keys_missing_file(tmp_path): - skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") - assert skip == set() and n_skip == 0 and n_retry == 0 - - _d0 = tmp_path / "test_load_resume_skip_keys_summary" - _d0.mkdir() - test_load_resume_skip_keys_summary(_d0) - _d1 = tmp_path / "test_load_resume_skip_keys_n_retry_ignores_later_success" - _d1.mkdir() - test_load_resume_skip_keys_n_retry_ignores_later_success(_d1) - _d2 = tmp_path / "test_load_resume_skip_keys_label_mismatch" - _d2.mkdir() - test_load_resume_skip_keys_label_mismatch(_d2) - _d3 = tmp_path / "test_load_resume_skip_keys_battery_model_driver_mismatch" - _d3.mkdir() - test_load_resume_skip_keys_battery_model_driver_mismatch(_d3) - _d4 = tmp_path / "test_load_resume_skip_keys_truncated_json" - _d4.mkdir() - test_load_resume_skip_keys_truncated_json(_d4, capsys) - _d5 = tmp_path / "test_load_resume_skip_keys_missing_file" - _d5.mkdir() - test_load_resume_skip_keys_missing_file(_d5) - - -def test_resume_behaviours(tmp_path): - def test_resume_identity_uses_resolved_model_not_tier_label(tmp_path): - p = tmp_path / "tiered.jsonl" - p.write_text( - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "local", - "model": "provider-reported-id", - "requested_model": "standard", - "requested_tier": "standard", - "resolved_model": "old-standard-id", - "error": None, - } - ) - + "\n", - encoding="utf-8", + + "\n", + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") + with pytest.raises(SystemExit, match="driver"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") + # Missing keys on older rows: pass (back-compat) + p2 = tmp_path / "old.jsonl" + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0, "local") in skip + + +def _load_resume_skip_keys_truncated_json(tmp_path, capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + + "\n" + + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + err = capsys.readouterr().err + assert "invalid JSON" in err + + +def _load_resume_skip_keys_missing_file(tmp_path, _capsys): + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") + assert skip == set() and n_skip == 0 and n_retry == 0 + + +@pytest.mark.parametrize( + "case", + case_params( + _load_resume_skip_keys_summary, + _load_resume_skip_keys_n_retry_ignores_later_success, + _load_resume_skip_keys_label_mismatch, + _load_resume_skip_keys_battery_model_driver_mismatch, + _load_resume_skip_keys_truncated_json, + _load_resume_skip_keys_missing_file, + ), +) +def test_load_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) + + +def _resume_identity_uses_resolved_model_not_tier_label(tmp_path): + p = tmp_path / "tiered.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "model": "provider-reported-id", + "requested_model": "standard", + "requested_tier": "standard", + "resolved_model": "old-standard-id", + "error": None, + } ) + + "\n", + encoding="utf-8", + ) - skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") - assert skip == {("R1", 0, "local")} - with pytest.raises(SystemExit, match="model"): - load_resume_skip_keys(p, label="local", model="new-standard-id") - - def test_resume_skips_meta_and_mismatch_checks_it(tmp_path): - p = tmp_path / "out.jsonl" - p.write_text( - "\n".join( - [ - json.dumps( - { - "row_type": "meta", - "label": "candidate", - "battery": "bbbbbbbbbbbb", - "model": "sonnet", - "driver": "claude-cli", - } - ), - json.dumps( - { - "task_id": "R1", - "rep": 0, - "label": "candidate", - "error": None, - "error_class": None, - "success": True, - } - ), - ] - ) - + "\n", - encoding="utf-8", - ) - skip, n_skip, n_retry = load_resume_skip_keys( - p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") + assert skip == {("R1", 0, "local")} + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", model="new-standard-id") + + +def _resume_skips_meta_and_mismatch_checks_it(tmp_path): + p = tmp_path / "out.jsonl" + p.write_text( + "\n".join( + [ + json.dumps( + { + "row_type": "meta", + "label": "candidate", + "battery": "bbbbbbbbbbbb", + "model": "sonnet", + "driver": "claude-cli", + } + ), + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "candidate", + "error": None, + "error_class": None, + "success": True, + } + ), + ] ) - assert skip == {("R1", 0, "candidate")} - assert n_skip == 1 and n_retry == 0 + + "\n", + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys( + p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + ) + assert skip == {("R1", 0, "candidate")} + assert n_skip == 1 and n_retry == 0 - with pytest.raises(SystemExit, match="battery"): - load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") - _d0 = tmp_path / "test_resume_identity_uses_resolved_model_not_tier_label" - _d0.mkdir() - test_resume_identity_uses_resolved_model_not_tier_label(_d0) - _d1 = tmp_path / "test_resume_skips_meta_and_mismatch_checks_it" - _d1.mkdir() - test_resume_skips_meta_and_mismatch_checks_it(_d1) + +@pytest.mark.parametrize( + "case", + case_params(_resume_identity_uses_resolved_model_not_tier_label, _resume_skips_meta_and_mismatch_checks_it), +) +def test_resume_behaviours(case, tmp_path): + case(tmp_path) -def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypatch): +def test_run_live_resume_retries_infra_and_unexpected_skips_but_not_plan_gates(tmp_path: Path, monkeypatch): out = tmp_path / "resume.jsonl" - # Pre-write: completed R1/0 + infra R2/0 (same label/battery/model/driver as this run). + # Pre-write a completed result, infra error, expected skip, and unexpected skip. # Battery is computed from the task list below — seed the file after we know it, # or write rows without battery (back-compat) and only check skip/retry behavior. prior = [ @@ -245,6 +268,22 @@ def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypat "error_class": "infra_seed", "success": False, }, + { + "task_id": "C1", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "skipped": "env:plan-gated:customers", + }, + { + "task_id": "C2", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "skipped": "env:fixture-collision:release_tags:eval-rc1", + }, ] out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") @@ -252,10 +291,10 @@ def test_run_live_resume_skips_completed_retries_infra(tmp_path: Path, monkeypat monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) seed_calls: list[str] = [] - def ok_seed(plane, run_id, needs, ctx): + def ok_seed(plane, run_id, needs, ctx, task_id=None): # Infer task from empty ctx; runner sets project for verify path. ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) - seed_calls.append(run_id) + seed_calls.append(str(task_id)) monkeypatch.setattr(runner_live, "seed", ok_seed) monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) @@ -281,9 +320,6 @@ async def verify_ok(plane, ctx, run): "id": "R1", "prompt": "do {project}", "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, "needs": set(), "verify": verify_ok, }, @@ -291,9 +327,20 @@ async def verify_ok(plane, ctx, run): "id": "R2", "prompt": "do {project}", "tags": set(), - "optimal_tools": {"list_work_items"}, - "alternate_tools": set(), - "optimal_calls": 1, + "needs": set(), + "verify": verify_ok, + }, + { + "id": "C1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + }, + { + "id": "C2", + "prompt": "do {project}", + "tags": set(), "needs": set(), "verify": verify_ok, }, @@ -311,16 +358,18 @@ async def verify_ok(plane, ctx, run): ) ) assert rc == 0 - # Only R2 should have been re-seeded/run (R1 completed → RESUME_SKIP). - assert len(seed_calls) == 1 + # R1 completed and C1 was plan-gated. R2 infra and C2 collision are retried. + assert seed_calls == ["R2", "C2"] data = _data_rows(out) - # prior 2 + 1 new R2 row (meta may also exist if file was empty — it wasn't) - assert len(data) == 3 - new_r2 = data[-1] + # Prior four + two retries (meta may also exist if file was empty — it wasn't). + assert len(data) == 6 + new_r2, new_c2 = data[-2:] assert new_r2["task_id"] == "R2" assert new_r2["success"] is True assert new_r2["error_class"] is None assert new_r2["final_text"] == "done" + assert new_c2["task_id"] == "C2" + assert new_c2["success"] is True def test_make_run_meta_row_and_write_once(tmp_path: Path): From 7fe80ec4d051315d2e18d838c2602b1d913a8503 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 18:32:58 +0530 Subject: [PATCH 35/93] Keep the call trace honest about what it dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sidecar marked a trace complete while silently discarding corrupted rows: a torn final line set a flag, but a corrupted mid-stream line was dropped by a bare continue with no counter. That matters more than it did before, because provenance now matches a sentinel in the recorded trace — a dropped line can be the response carrying it, failing a correct agent with no diagnosis. Skipped rows are counted and make the trace incomplete, and provenance distinguishes "trace incomplete" from "sentinel not observed". Result rows are copied by reflection over a field list, with a parity test that fails when a field is added and not copied; result_tokens_skipped_reason had been declared, serialized and deserialized but never assigned. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/cli/claude.py | 2 +- evals/drivers/cli/sidecar.py | 20 +- evals/drivers/driver.py | 60 + evals/proxy.py | 52 +- evals/results.py | 187 +-- evals/tool_names.py | 9 +- tests/evals/drivers/test_api_driver.py | 725 ++++++------ tests/evals/drivers/test_cli_driver.py | 1005 +++++++++------- tests/evals/drivers/test_vendors.py | 1278 +++++++++++---------- tests/fixtures/evals_schema_v0_rows.jsonl | 4 +- 10 files changed, 1845 insertions(+), 1497 deletions(-) diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index f6c1c54e..7213e6c3 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -171,7 +171,7 @@ def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]] """Parse ``tool_use`` blocks from a Claude Code session JSONL transcript. Returns tagged calls (``origin`` plane|client). Use - ``split_plane_and_client_calls`` before classification. + ``split_plane_and_client_calls`` before counting. """ calls: list[dict[str, Any]] = [] if not transcript_path.is_file(): diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 8786b518..dabec415 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -18,12 +18,15 @@ def proxy_wrap_server_command( sidecar_path: Path, python_bin: str | None = None, record_result_payloads: bool = False, + evidence_path: Path | None = None, ) -> list[str]: """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" py = python_bin or sys.executable command = [py, "-m", "evals.proxy", "--log", str(sidecar_path)] if record_result_payloads: command.append("--record-result-payloads") + if evidence_path is not None: + command.extend(["--evidence-file", str(evidence_path)]) return [*command, "--", *real_command] @@ -50,12 +53,14 @@ def load_proxy_sidecar( Status keys: - missing / empty / complete / incomplete - torn_line: final line failed to parse + - skipped_rows: non-final rows that could not produce a call or metadata row - meta: proxy_meta row if present - pending_left: from meta when present """ status: dict[str, Any] = { "state": "missing", "torn_line": False, + "skipped_rows": 0, "meta": None, "pending_left": None, } @@ -75,6 +80,7 @@ def load_proxy_sidecar( calls: list[dict[str, Any]] = [] meta: dict[str, Any] | None = None torn = False + skipped_rows = 0 for i, line in enumerate(lines): s = line.strip() if not s: @@ -86,14 +92,17 @@ def load_proxy_sidecar( if i == len(lines) - 1: torn = True break + skipped_rows += 1 continue if not isinstance(row, dict): + skipped_rows += 1 continue if row.get("row_type") == "proxy_meta": meta = row continue tool = row.get("tool") if not tool: + skipped_rows += 1 continue call = { "tool": str(tool), @@ -107,23 +116,27 @@ def load_proxy_sidecar( # Optional in new sidecars; old payload-free rows remain valid. if isinstance(row.get("result_text"), str): call["result_text"] = row["result_text"] + if isinstance(row.get("observed_sentinels"), list): + call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] calls.append(call) # Score order must match request seq, not response-append order. calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) status["torn_line"] = torn + status["skipped_rows"] = skipped_rows status["meta"] = meta if meta is not None: status["pending_left"] = meta.get("pending_left") status["pumps_alive"] = bool(meta.get("pumps_alive")) incomplete = bool( torn + or skipped_rows > 0 or meta is None or (meta is not None and int(meta.get("pending_left") or 0) > 0) or (meta is not None and bool(meta.get("pumps_alive"))) ) - if not calls and not meta and not torn: + if not calls and not meta and not torn and skipped_rows == 0: status["state"] = "empty" elif incomplete: status["state"] = "incomplete" @@ -146,8 +159,8 @@ def apply_proxy_sidecar( ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. - Incomplete sidecar (torn line, missing meta, pending_left>0) yields to the - CLI trace when the CLI has *more* plane calls. Returns + Incomplete sidecar (torn/skipped row, missing meta, pending_left>0) yields + to the CLI trace when the CLI has *more* plane calls. Returns ``(plane_calls, client_calls, call_source)``. """ proxy_calls, status = load_proxy_sidecar(sidecar_path) @@ -159,6 +172,7 @@ def apply_proxy_sidecar( notes.append( "proxy_sidecar_incomplete" + (":torn" if status.get("torn_line") else "") + + (f":skipped_rows={status.get('skipped_rows')}" if status.get("skipped_rows") else "") + (":no_meta" if status.get("meta") is None else "") + (f":pending_left={status.get('pending_left')}" if status.get("pending_left") else "") + (":pumps_alive" if status.get("pumps_alive") else "") diff --git a/evals/drivers/driver.py b/evals/drivers/driver.py index 1edeea9e..7ecfab8f 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/driver.py @@ -33,8 +33,16 @@ apply_proxy_sidecar, ensure_proxy_pythonpath, harvest_proxy_after_cli_timeout, + load_proxy_sidecar, proxy_wrap_server_command, ) +from evals.evidence import ( + configured_evidence_labels, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_sentinel_labels, + write_evidence_config, +) from evals.results import AgentRun, Usage from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens @@ -189,6 +197,8 @@ def run_task( *, system: str | None = None, cwd: Path | None = None, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, ) -> AgentRun: if not model: raise ValueError("the API driver requires a model ID") @@ -202,6 +212,8 @@ def run_task( max_turns=max_turns, system=system, cwd=cwd, + evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, ) ) @@ -214,8 +226,13 @@ async def _run_task( max_turns: int, system: str | None, cwd: Path | None, + evidence_sentinels: dict[str, Any] | None, + evidence_targets: dict[str, Any] | None, ) -> AgentRun: backend = self._make_backend(model) + evidence = normalize_evidence_sentinels(evidence_sentinels) + targets = normalize_evidence_targets(evidence_targets) + evidence_active = bool(configured_evidence_labels(evidence, targets)) calls: list[dict[str, Any]] = [] pending_results: list[tuple[int, str]] = [] usage_per_iteration: list[Usage] = [] @@ -303,6 +320,13 @@ async def _run_task( calls[idx]["result_kind"] = result.kind calls[idx]["is_error"] = result.is_error calls[idx]["duration_ms"] = duration_ms + if evidence_active: + calls[idx]["observed_sentinels"] = observed_sentinel_labels( + result.text, + evidence, + request_args=calls[idx]["args"], + evidence_targets=targets, + ) pending_results.append((idx, result.text)) if matched_ids != set(call_indices) or len(call_indices) != len(turn.tool_calls): result_pair_mismatch = True @@ -366,6 +390,7 @@ async def _run_task( result_pair_mismatch=result_pair_mismatch, token_count_failures=token_count_failures, result_tokens_estimated=result_tokens_estimated, + evidence_trace_available=evidence_active, provider=str(backend.provider), model=str(backend.actual_model), requested_model=model, @@ -507,6 +532,8 @@ def run_task( *, system: str | None = None, cwd: Path | None = None, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, ) -> AgentRun: """Run one CLI task using the shared configuration/proxy/timeout flow.""" task_cwd = (cwd or REPO_ROOT).resolve() @@ -520,16 +547,24 @@ def run_task( child_env = { key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") } + evidence = normalize_evidence_sentinels(evidence_sentinels) + targets = normalize_evidence_targets(evidence_targets) + evidence_active = bool(configured_evidence_labels(evidence, targets)) real_command = ( list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] ) server_command = real_command if self.use_proxy: + evidence_path = None + if evidence_active: + evidence_path = temp_dir / "proxy-evidence.json" + write_evidence_config(evidence_path, evidence, targets) server_command = proxy_wrap_server_command( real_command, sidecar_path=sidecar, python_bin=self.python_bin, record_result_payloads=self.record_result_payloads, + evidence_path=evidence_path, ) child_env = ensure_proxy_pythonpath(child_env) @@ -566,6 +601,17 @@ def run_task( sidecar, notes, ) + evidence_available = False + if evidence_active and call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + meta = status.get("meta") if isinstance(status, dict) else None + evidence_available = bool( + status.get("state") == "complete" + and isinstance(meta, dict) + and meta.get("evidence_trace_available") + ) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") return AgentRun( calls=calls, client_tool_calls=client_calls, @@ -577,6 +623,7 @@ def run_task( call_source=call_source, hit_max_turns=False, wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, experimental=self.experimental, notes=notes, ) @@ -607,6 +654,18 @@ def run_task( if proxy_source == "proxy": output.call_source = "proxy" + evidence_available = False + if evidence_active and output.call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + meta = status.get("meta") if isinstance(status, dict) else None + evidence_available = bool( + status.get("state") == "complete" + and isinstance(meta, dict) + and meta.get("evidence_trace_available") + ) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") + self.finalize_run(proc, output=output, notes=notes) return AgentRun( calls=output.calls, @@ -620,6 +679,7 @@ def run_task( call_source=output.call_source, hit_max_turns=output.hit_max_turns, wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, experimental=self.experimental, notes=notes, ) diff --git a/evals/proxy.py b/evals/proxy.py index 7cfce9fb..bb40e52a 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -19,6 +19,15 @@ from pathlib import Path from typing import Any +from evals.evidence import ( + EVIDENCE_SENTINELS_ENV, + configured_evidence_labels, + consume_evidence_config, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_sentinel_labels, +) + # Single post-EOF / child-exit deadline for the whole shutdown sequence. SHUTDOWN_DEADLINE_S = 10.0 READ_CHUNK = 65536 @@ -45,6 +54,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Also store serialized tool-result text (off by default; may contain workspace data)", ) + p.add_argument( + "--evidence-file", + type=Path, + help="One-shot target-evidence configuration consumed before the MCP child starts", + ) p.add_argument( "command", nargs=argparse.REMAINDER, @@ -92,6 +106,9 @@ def scrub_child_pythonpath(env: dict[str, str] | None = None) -> dict[str, str]: ``plane_mcp`` from its own venv). """ base = dict(env if env is not None else os.environ) + # Matching configuration belongs only to the recorder. The real Plane MCP server + # neither needs nor receives hidden sentinel values. + base.pop(EVIDENCE_SENTINELS_ENV, None) root = str(REPO_ROOT) raw = base.get("PYTHONPATH", "") if not raw: @@ -114,9 +131,19 @@ class SidecarRecorder: last sidecar line even if daemon pumps keep running briefly. """ - def __init__(self, log_path: Path, *, record_result_payloads: bool = False) -> None: + def __init__( + self, + log_path: Path, + *, + record_result_payloads: bool = False, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + ) -> None: self.log_path = log_path self.record_result_payloads = record_result_payloads + self.evidence_sentinels = normalize_evidence_sentinels(evidence_sentinels) + self.evidence_targets = normalize_evidence_targets(evidence_targets) + self.evidence_active = bool(configured_evidence_labels(self.evidence_sentinels, self.evidence_targets)) self._lock = threading.Lock() self._pending: dict[Any, dict[str, Any]] = {} self._seq = 0 @@ -226,6 +253,14 @@ def on_server_message(self, obj: dict[str, Any]) -> None: "duration_ms": duration_ms, "seq": pending["seq"], } + if self.evidence_active: + # Persist labels only. The matching values and result body stay in memory. + row["observed_sentinels"] = observed_sentinel_labels( + result_text, + self.evidence_sentinels, + request_args=pending["args"], + evidence_targets=self.evidence_targets, + ) if self.record_result_payloads: row["result_text"] = result_text self._append(row) @@ -246,6 +281,7 @@ def write_meta(self) -> None: "pending_left": len(self._pending), "child_killed": self.child_killed, "pumps_alive": self.pumps_alive, + "evidence_trace_available": self.evidence_active, } line = json.dumps(row, default=str, ensure_ascii=False) + "\n" with self.log_path.open("a", encoding="utf-8") as fh: @@ -395,6 +431,8 @@ def run_proxy( log_path: Path, *, record_result_payloads: bool = False, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, ) -> int: """Spawn ``command`` as the real MCP server and relay with recording. @@ -403,7 +441,12 @@ def run_proxy( crash paths. Pump threads are daemon so a blocked write cannot hold the process past the shutdown deadline. """ - recorder = SidecarRecorder(log_path, record_result_payloads=record_result_payloads) + recorder = SidecarRecorder( + log_path, + record_result_payloads=record_result_payloads, + evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, + ) child: subprocess.Popen[bytes] | None = None # Scrub repo PYTHONPATH so the real server does not import from this tree. child_env = scrub_child_pythonpath() @@ -571,6 +614,8 @@ def run_proxy( try: recorder.write_meta() except Exception as exc: + # Safe to continue: no meta marks the sidecar incomplete, so the parent + # driver rejects it as authoritative and falls back to the CLI trace. print(f"evals.proxy: failed to write proxy_meta: {exc}", file=sys.stderr) @@ -585,10 +630,13 @@ def main(argv: list[str] | None = None) -> int: # Already a session leader, or platform forbids setsid — continue. pass args = parse_args(argv) + evidence_sentinels, evidence_targets = consume_evidence_config(args.evidence_file) return run_proxy( list(args.command), Path(args.log), record_result_payloads=bool(args.record_result_payloads), + evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, ) diff --git a/evals/results.py b/evals/results.py index 97bfbd0a..c9c1432f 100644 --- a/evals/results.py +++ b/evals/results.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -from collections.abc import Callable from dataclasses import dataclass, field from typing import Any, Literal @@ -14,7 +13,63 @@ ) from evals.tool_names import split_plane_and_client_calls -RESULT_SCHEMA_VERSION = 1 +RESULT_SCHEMA_VERSION = 3 + +# ``apply_agent_result`` owns this explicit partition. A reflection test compares +# it with every TaskResult dataclass field so additions cannot disappear silently. +AGENT_RESULT_COPY_FIELDS = ( + "final_text", + "stop_reason", + "provider_stop_reason", + "hit_max_iterations", + "result_pair_mismatch", + "token_count_failures", + "result_tokens_estimated", + "calls", + "num_calls", + "errored_calls", + "total_result_tokens", + "usage_per_iteration", + "cum_input_tokens", + "cum_input_tokens_reason", + "wall_time_s", + "client_tool_calls", + "client_tool_call_count", + "result_tokens_mode", + "result_token_count_method", + "usage_scope", + "call_source", + "evidence_trace_available", + "driver_raw_ref", + "driver_notes", + "usage", + "usage_total", + "result_tokens_skipped_reason", +) +AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS = ("provider", "model", "requested_model") +TASK_RESULT_HARNESS_FIELDS = ( + "schema_version", + "row_type", + "run_id", + "ts", + "git_sha", + "battery", + "label", + "driver", + "server", + "requested_tier", + "resolved_model", + "task_id", + "author", + "rep", + "expected_rows", + "success", + "verify_note", + "skipped", + "error", + "error_class", + "cleanup_error", +) @dataclass(frozen=True, slots=True) @@ -32,7 +87,6 @@ class CallRecord: """Persisted metrics for one Plane or client tool call.""" tool: str - classification: str | None = None args_chars: int = 0 result_tokens: int | None = None result_chars: int = 0 @@ -44,19 +98,21 @@ class CallRecord: action: str | None = None raw_tool: str | None = None result_tokens_skipped: str | None = None + # None means the response was not checked; [] means checked with no match. + observed_sentinels: list[str] | None = None @dataclass class AgentRun: """Normalized result of one agent task execution.""" - # Plane MCP tools only for classification: {tool, args, origin='plane', raw_tool?} + # Plane MCP tools only: {tool, args, origin='plane', raw_tool?} calls: list[dict[str, Any]] final_text: str usage: Usage | dict[str, Any] | None stopped_reason: str raw_ref: str | None = None - # Client/harness built-ins (ToolSearch, Bash, …) — excluded from mispick metrics + # Client/harness built-ins (ToolSearch, Bash, …) are retained separately. client_tool_calls: list[dict[str, Any]] = field(default_factory=list) # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens usage_total: dict[str, Any] | None = None @@ -75,6 +131,7 @@ class AgentRun: # means at least one result used the shared character estimate. None lets # the common row mapper determine the status from the recorded calls. result_tokens_estimated: bool | None = None + evidence_trace_available: bool = False provider: str | None = None model: str | None = None requested_model: str | None = None @@ -89,7 +146,9 @@ class TaskResult: schema_version 0 marks rows written before this type existed; from_row defaults every field added since. Version 1 defines wall_time_s as CLI invocation time only — earlier - Claude/Antigravity/OpenCode rows also include a few ms of harness setup. + Claude/Antigravity/OpenCode rows also include a few ms of harness setup. Version 2 adds + run-completeness metadata and cleanup failure recording. Version 3 records only + response-evidence labels (never Plane response bodies) plus trace availability. """ schema_version: int = RESULT_SCHEMA_VERSION @@ -109,11 +168,13 @@ class TaskResult: task_id: str = "" author: str = "" rep: int = 0 + expected_rows: int = 0 success: bool = False verify_note: str = "" skipped: str | None = None error: str | None = None error_class: str | None = None + cleanup_error: str | None = None final_text: str = "" stop_reason: str | None = None provider_stop_reason: str | None = None @@ -124,8 +185,6 @@ class TaskResult: calls: list[CallRecord] = field(default_factory=list) num_calls: int = 0 errored_calls: int = 0 - alternate_calls: int | None = 0 - out_of_set_calls: int | None = 0 total_result_tokens: int = 0 usage_per_iteration: list[Usage] = field(default_factory=list) cum_input_tokens: int | None = 0 @@ -137,6 +196,7 @@ class TaskResult: result_token_count_method: str | None = None usage_scope: str | None = None call_source: str | None = None + evidence_trace_available: bool = False driver_raw_ref: str | None = None driver_notes: list[str] = field(default_factory=list) usage: Usage | dict[str, Any] | None = None @@ -145,39 +205,12 @@ class TaskResult: def apply_agent_result(self, agent: TaskResult) -> None: """Copy the driver-owned portion of an agent result onto this task row.""" - self.final_text = agent.final_text - self.stop_reason = agent.stop_reason - self.provider_stop_reason = agent.provider_stop_reason - self.hit_max_iterations = agent.hit_max_iterations - self.result_pair_mismatch = agent.result_pair_mismatch - self.token_count_failures = agent.token_count_failures - self.result_tokens_estimated = agent.result_tokens_estimated - self.calls = agent.calls - self.num_calls = agent.num_calls - self.errored_calls = agent.errored_calls - self.alternate_calls = agent.alternate_calls - self.out_of_set_calls = agent.out_of_set_calls - self.total_result_tokens = agent.total_result_tokens - self.usage_per_iteration = agent.usage_per_iteration - self.cum_input_tokens = agent.cum_input_tokens - self.cum_input_tokens_reason = agent.cum_input_tokens_reason - self.wall_time_s = agent.wall_time_s - self.client_tool_calls = agent.client_tool_calls - self.client_tool_call_count = agent.client_tool_call_count - self.result_tokens_mode = agent.result_tokens_mode - self.result_token_count_method = agent.result_token_count_method - self.usage_scope = agent.usage_scope - self.call_source = agent.call_source - self.driver_raw_ref = agent.driver_raw_ref - self.driver_notes = agent.driver_notes - self.usage = agent.usage - self.usage_total = agent.usage_total - if agent.provider is not None: - self.provider = agent.provider - if agent.model is not None: - self.model = agent.model - if agent.requested_model is not None: - self.requested_model = agent.requested_model + for field_name in AGENT_RESULT_COPY_FIELDS: + setattr(self, field_name, getattr(agent, field_name)) + for field_name in AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS: + value = getattr(agent, field_name) + if value is not None: + setattr(self, field_name, value) def to_row(self) -> dict[str, Any]: """Serialize the versioned persisted JSONL row schema. @@ -200,7 +233,6 @@ def usage_row(item: Usage) -> dict[str, int]: for call in self.calls: item: dict[str, Any] = { "tool": call.tool, - "class": call.classification, "args_chars": call.args_chars, "result_tokens": call.result_tokens, "result_chars": call.result_chars, @@ -215,6 +247,8 @@ def usage_row(item: Usage) -> dict[str, int]: item["action"] = call.action if call.result_tokens_skipped is not None: item["result_tokens_skipped"] = call.result_tokens_skipped + if call.observed_sentinels is not None: + item["observed_sentinels"] = list(call.observed_sentinels) calls.append(item) client_calls = [ @@ -242,11 +276,13 @@ def usage_row(item: Usage) -> dict[str, int]: "task_id": self.task_id, "author": self.author, "rep": self.rep, + "expected_rows": self.expected_rows, "success": self.success, "verify_note": self.verify_note, "skipped": self.skipped, "error": self.error, "error_class": self.error_class, + "cleanup_error": self.cleanup_error, "final_text": self.final_text, "stop_reason": self.stop_reason, "provider_stop_reason": self.provider_stop_reason, @@ -257,8 +293,6 @@ def usage_row(item: Usage) -> dict[str, int]: "calls": calls, "num_calls": self.num_calls, "errored_calls": self.errored_calls, - "alternate_calls": self.alternate_calls, - "out_of_set_calls": self.out_of_set_calls, "total_result_tokens": self.total_result_tokens, "usage_per_iteration": [usage_row(item) for item in self.usage_per_iteration], "cum_input_tokens": self.cum_input_tokens, @@ -270,6 +304,7 @@ def usage_row(item: Usage) -> dict[str, int]: "result_token_count_method": self.result_token_count_method, "usage_scope": self.usage_scope, "call_source": self.call_source, + "evidence_trace_available": self.evidence_trace_available, "driver_raw_ref": self.driver_raw_ref, "driver_notes": list(self.driver_notes), "usage": usage_row(self.usage) if isinstance(self.usage, Usage) else self.usage, @@ -283,7 +318,7 @@ def usage_row(item: Usage) -> dict[str, int]: @classmethod def from_row(cls, row: dict[str, Any]) -> TaskResult: - """Read current or pre-versioned persisted rows with stable defaults.""" + """Read a persisted row, retaining defaults for unrelated older fields.""" raw_calls = row.get("calls") if isinstance(row.get("calls"), list) else [] calls: list[CallRecord] = [] for raw in raw_calls: @@ -292,7 +327,6 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: calls.append( CallRecord( tool=str(raw.get("tool") or ""), - classification=(str(raw["class"]) if raw.get("class") is not None else None), args_chars=int(raw.get("args_chars") or 0), result_tokens=(int(raw["result_tokens"]) if raw.get("result_tokens") is not None else None), result_chars=int(raw.get("result_chars") or 0), @@ -311,6 +345,11 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: result_tokens_skipped=( str(raw["result_tokens_skipped"]) if raw.get("result_tokens_skipped") is not None else None ), + observed_sentinels=( + [str(value) for value in raw["observed_sentinels"]] + if isinstance(raw.get("observed_sentinels"), list) + else None + ), ) ) @@ -342,8 +381,6 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ) ) - alternate_default = sum(1 for call in calls if call.classification == "alternate") - out_of_set_default = sum(1 for call in calls if call.classification == "out_of_set") return cls( schema_version=int(row.get("schema_version") or 0), row_type=(str(row["row_type"]) if row.get("row_type") is not None else None), @@ -362,11 +399,13 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: task_id=str(row.get("task_id") or ""), author=str(row.get("author") or ""), rep=int(row.get("rep") or 0), + expected_rows=int(row.get("expected_rows") or 0), success=bool(row.get("success")), verify_note=str(row.get("verify_note") or ""), skipped=(str(row["skipped"]) if row.get("skipped") is not None else None), error=(str(row["error"]) if row.get("error") is not None else None), error_class=(str(row["error_class"]) if row.get("error_class") is not None else None), + cleanup_error=(str(row["cleanup_error"]) if row.get("cleanup_error") is not None else None), final_text=str(row.get("final_text") or ""), stop_reason=(str(row["stop_reason"]) if row.get("stop_reason") is not None else None), provider_stop_reason=( @@ -385,20 +424,6 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: if row.get("errored_calls") is not None else sum(1 for call in calls if call.is_error) ), - alternate_calls=( - int(row["alternate_calls"]) - if row.get("alternate_calls") is not None - else None - if "alternate_calls" in row - else alternate_default - ), - out_of_set_calls=( - int(row["out_of_set_calls"]) - if row.get("out_of_set_calls") is not None - else None - if "out_of_set_calls" in row - else out_of_set_default - ), total_result_tokens=int( row.get("total_result_tokens") if row.get("total_result_tokens") is not None @@ -422,6 +447,7 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ), usage_scope=(str(row["usage_scope"]) if row.get("usage_scope") is not None else None), call_source=(str(row["call_source"]) if row.get("call_source") is not None else None), + evidence_trace_available=bool(row.get("evidence_trace_available")), driver_raw_ref=(str(row["driver_raw_ref"]) if row.get("driver_raw_ref") is not None else None), driver_notes=[str(item) for item in row.get("driver_notes") or []], usage=row.get("usage") if isinstance(row.get("usage"), dict) else None, @@ -436,16 +462,12 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: def agent_run_to_task_result( run: AgentRun, - *, - optimal: set[str], - alternate: set[str], - classify: Callable[[str, set[str], set[str]], str], ) -> TaskResult: """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. - Only Plane MCP tools count toward num_calls and mispicks; client built-ins go to - client_tool_calls. CLI drivers never fill cum_input_tokens from bare usage.input_tokens - — under Claude Code that is uncached-only and misreads cached runs as ~10 tokens. + Only Plane MCP tools count toward num_calls; client built-ins go to client_tool_calls. + CLI drivers never fill cum_input_tokens from bare usage.input_tokens — under Claude + Code that is uncached-only and misreads cached runs as ~10 tokens. """ # Re-split in case callers passed a mixed list plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) @@ -484,7 +506,6 @@ def agent_run_to_task_result( rec = CallRecord( tool=str(tool), - classification=classify(str(tool), optimal, alternate), args_chars=args_chars, result_tokens=result_tokens, result_chars=result_chars, @@ -493,6 +514,11 @@ def agent_run_to_task_result( result_tokens_estimated=bool(estimated), result_token_count_method=str(count_method), duration_ms=c.get("duration_ms"), + observed_sentinels=( + [str(value) for value in c["observed_sentinels"]] + if isinstance(c.get("observed_sentinels"), list) + else None + ), ) # Action-dispatch surfaces: the action arg IS the second half of the # tool choice — keep it (args content is otherwise not persisted). @@ -522,8 +548,6 @@ def agent_run_to_task_result( stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" errored = sum(1 for c in calls if c.is_error) - alternate_n = sum(1 for c in calls if c.classification == "alternate") - out_of_set_n = sum(1 for c in calls if c.classification == "out_of_set") # CLI path: never write misleading cum_input_tokens from uncached-only field. # usage_total is driver-owned — do not re-derive it here (Claude vs Codex @@ -578,8 +602,6 @@ def agent_run_to_task_result( client_tool_calls=client_tool_calls, client_tool_call_count=len(client_tool_calls), errored_calls=errored, - alternate_calls=alternate_n, - out_of_set_calls=out_of_set_n, total_result_tokens=sum(int(c.result_tokens or 0) for c in calls), usage_per_iteration=usage_per_iteration, cum_input_tokens=cum_input, @@ -595,6 +617,7 @@ def agent_run_to_task_result( result_token_count_method=result_token_count_method, usage_scope=run.usage_scope, call_source=run.call_source, + evidence_trace_available=run.evidence_trace_available, driver_raw_ref=run.raw_ref, driver_notes=list(run.notes), usage=run.usage, @@ -607,22 +630,16 @@ def agent_run_to_task_result( def agent_run_to_harness_dict( run: AgentRun, - *, - optimal: set[str], - alternate: set[str], - classify: Callable[[str, set[str], set[str]], str], ) -> dict[str, Any]: - """Compatibility wrapper returning the public persisted-row dictionary.""" - return agent_run_to_task_result( - run, - optimal=optimal, - alternate=alternate, - classify=classify, - ).to_row() + """Map an agent run to the public persisted-row dictionary.""" + return agent_run_to_task_result(run).to_row() __all__ = [ + "AGENT_RESULT_COPY_FIELDS", + "AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS", "RESULT_SCHEMA_VERSION", + "TASK_RESULT_HARNESS_FIELDS", "AgentRun", "CallRecord", "TaskResult", diff --git a/evals/tool_names.py b/evals/tool_names.py index cb890459..e82d682e 100644 --- a/evals/tool_names.py +++ b/evals/tool_names.py @@ -2,9 +2,8 @@ Agent CLIs expose MCP tools under a vendor prefix (``mcp__plane__list_work_items``) and mix them with their own built-ins (``Bash``, ``ToolSearch``). Drivers and the -result mapper both have to tell those apart before anything is classified or -counted, so this sits beside the result schema rather than inside one driver -package. +result mapper both have to tell those apart before calls are counted, so this +sits beside the result schema rather than inside one driver package. """ from __future__ import annotations @@ -20,7 +19,7 @@ def strip_mcp_prefix(name: str) -> str: - """Strip Claude/Codex MCP tool name prefixes for classification. + """Strip Claude/Codex MCP tool name prefixes. Examples: mcp__plane__list_work_items → list_work_items @@ -48,7 +47,7 @@ def is_plane_mcp_tool(name: str) -> bool: def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: - """Tag a tool call as plane (classifiable) or client (excluded from mispicks).""" + """Tag a tool call as Plane or client-owned.""" raw = str(name or "") if not isinstance(args, dict): args = {"_raw": args} diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 64257077..85357ddb 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -26,7 +26,9 @@ resolve_backend_model, unregister_backend, ) +from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.token_counting import estimate_result_tokens +from tests.evals.conftest import case_params class FakeBackend: @@ -97,13 +99,14 @@ async def session_factory(_params): ) -def run_driver(driver: ApiDriver, *, max_turns: int = 5): +def run_driver(driver: ApiDriver, *, max_turns: int = 5, evidence_sentinels=None): return driver.run_task( "do it", {"SAFE": "1"}, "fake-requested", max_turns, system="system", + evidence_sentinels=evidence_sentinels, ) @@ -181,198 +184,249 @@ async def session_factory(_params): assert run.usage_per_iteration == [Usage(7, 2, 0, 0)] -def test_api_driver_behaviours(): - def test_api_driver_multi_turn_tool_loop_and_usage_accumulation(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], - usage=Usage(10, 2, 3, 1), - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="", - tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], - usage=Usage(20, 4, 6, 0), - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="done", - tool_calls=[], - usage=Usage(30, 6, 9, 0), - stop_reason=StopReason.END_TURN, - provider_stop_reason="fake_done", - ), - ] - ) - session = FakeMcpSession( - [ - {"content": [{"type": "text", "text": "first result"}], "isError": False}, - {"content": [{"type": "text", "text": "second"}], "isError": True}, - ] - ) +def _api_driver_multi_turn_tool_loop_and_usage_accumulation(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], + usage=Usage(10, 2, 3, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], + usage=Usage(20, 4, 6, 0), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=Usage(30, 6, 9, 0), + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", + ), + ] + ) + session = FakeMcpSession( + [ + {"content": [{"type": "text", "text": "first result"}], "isError": False}, + {"content": [{"type": "text", "text": "second"}], "isError": True}, + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert session.initialized is True + assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] + assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] + assert run.final_text == "done" + assert run.stopped_reason == "end_turn" + assert run.cum_input_tokens == 60 + assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] + assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] + assert [call["result_tokens"] for call in run.calls] == [ + estimate_result_tokens(len("first result")), + estimate_result_tokens(len("second")), + ] + assert [call["is_error"] for call in run.calls] == [False, True] + assert run.result_tokens_estimated is True + assert run.token_count_failures == 0 + assert run.provider == "fake" + assert run.model == "fake-actual" + assert run.provider_stop_reason == "fake_done" + - run = run_driver(make_driver(backend, session)) - - assert session.initialized is True - assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] - assert backend.started is not None - assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] - assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] - assert run.final_text == "done" - assert run.stopped_reason == "end_turn" - assert run.cum_input_tokens == 60 - assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] - assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] - assert [call["result_tokens"] for call in run.calls] == [ - estimate_result_tokens(len("first result")), - estimate_result_tokens(len("second")), +def _api_driver_refusal_records_calls_but_executes_nothing(): + backend = FakeBackend( + [ + Turn( + text="declined", + tool_calls=[ToolCall("write-1", "write", {"value": "x"})], + usage=Usage(1, 1), + stop_reason=StopReason.REFUSAL, + ) ] - assert [call["is_error"] for call in run.calls] == [False, True] - assert run.result_tokens_estimated is True - assert run.token_count_failures == 0 - assert run.provider == "fake" - assert run.model == "fake-actual" - assert run.provider_stop_reason == "fake_done" - - def test_api_driver_refusal_records_calls_but_executes_nothing(): - backend = FakeBackend( - [ - Turn( - text="declined", - tool_calls=[ToolCall("write-1", "write", {"value": "x"})], - usage=Usage(1, 1), - stop_reason=StopReason.REFUSAL, - ) - ] - ) - session = FakeMcpSession() - - run = run_driver(make_driver(backend, session)) - - assert [call["tool"] for call in run.calls] == ["write"] - assert session.called == [] - assert backend.added_results == [] - assert run.stopped_reason == "refusal" - assert run.hit_max_turns is False - - def test_api_driver_pairs_results_by_id_not_ordinal(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), - ] - ) - # The fake session deliberately returns tagged results in reverse ID order. - session = FakeMcpSession( - [ - ToolResult(call_id="b", text="BBBB"), - ToolResult(call_id="a", text="A"), - ] - ) + ) + session = FakeMcpSession() - run = run_driver(make_driver(backend, session)) + run = run_driver(make_driver(backend, session)) - assert [call["result_chars"] for call in run.calls] == [1, 4] - assert run.result_pair_mismatch is False + assert [call["tool"] for call in run.calls] == ["write"] + assert session.called == [] + assert backend.added_results == [] + assert run.stopped_reason == "refusal" + assert run.hit_max_turns is False - def test_api_driver_flags_result_id_mismatch(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="done", - tool_calls=[], - usage=None, - stop_reason=StopReason.END_TURN, - ), - ] - ) - session = FakeMcpSession( - [ - ToolResult(call_id="b", text="BBBB"), - ToolResult(call_id="unknown", text="lost"), - ] - ) - run = run_driver(make_driver(backend, session)) +def _api_driver_pairs_results_by_id_not_ordinal(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + # The fake session deliberately returns tagged results in reverse ID order. + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="a", text="A"), + ] + ) + + run = run_driver(make_driver(backend, session)) - assert run.result_pair_mismatch is True - assert [call["result_chars"] for call in run.calls] == [0, 4] + assert [call["result_chars"] for call in run.calls] == [1, 4] + assert run.result_pair_mismatch is False - def test_api_driver_iteration_cap_only_flags_mid_tool_loop(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn( - text="must not be read", - tool_calls=[], - usage=None, - stop_reason=StopReason.END_TURN, - ), - ] - ) - session = FakeMcpSession([ToolResult(call_id="a", text="result")]) - run = run_driver(make_driver(backend, session), max_turns=1) +def _api_driver_flags_result_id_mismatch(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="unknown", text="lost"), + ] + ) + + run = run_driver(make_driver(backend, session)) - assert session.called == [("lookup", {"q": "a"})] - assert len(backend.added_results) == 1 - assert backend.num_turns == 1 - assert run.hit_max_turns is True - assert run.stopped_reason == "tool_use" + assert run.result_pair_mismatch is True + assert [call["result_chars"] for call in run.calls] == [0, 4] - def test_api_driver_clean_end_on_last_iteration_is_not_capped(): - backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) - run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) +def _api_driver_iteration_cap_only_flags_mid_tool_loop(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="must not be read", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession([ToolResult(call_id="a", text="result")]) + + run = run_driver(make_driver(backend, session), max_turns=1) - assert run.hit_max_turns is False - assert run.stopped_reason == "end_turn" + assert session.called == [("lookup", {"q": "a"})] + assert len(backend.added_results) == 1 + assert backend.num_turns == 1 + assert run.hit_max_turns is True + assert run.stopped_reason == "tool_use" - def test_api_driver_uses_optional_backend_token_counter(): - backend = FakeBackend( - [ - Turn( - text="", - tool_calls=[ToolCall("a", "lookup", {"q": "a"})], - usage=None, - stop_reason=StopReason.TOOL_USE, - ), - Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), - ] - ) - backend.count_tokens = lambda text: len(text) + 10 - run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) +def _api_driver_clean_end_on_last_iteration_is_not_capped(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) - assert run.calls[0]["result_tokens"] == 13 - assert run.result_tokens_estimated is False - assert run.token_count_failures == 0 + run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) - test_api_driver_multi_turn_tool_loop_and_usage_accumulation() - test_api_driver_refusal_records_calls_but_executes_nothing() - test_api_driver_pairs_results_by_id_not_ordinal() - test_api_driver_flags_result_id_mismatch() - test_api_driver_iteration_cap_only_flags_mid_tool_loop() - test_api_driver_clean_end_on_last_iteration_is_not_capped() - test_api_driver_uses_optional_backend_token_counter() + assert run.hit_max_turns is False + assert run.stopped_reason == "end_turn" + + +def _api_driver_uses_optional_backend_token_counter(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + backend.count_tokens = lambda text: len(text) + 10 + + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) + + assert run.calls[0]["result_tokens"] == 13 + assert run.result_tokens_estimated is False + assert run.token_count_failures == 0 + + +def _api_driver_records_only_matching_evidence_labels(): + sentinel = "hidden-target-fact-2f81a0cd" + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "unrelated"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("b", "lookup", {"q": "target"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="a", text="ordinary workspace response"), + ToolResult(call_id="b", text=f"state={sentinel}"), + ] + ) + + run = run_driver( + make_driver(backend, session), + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + ) + + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert "result_text" not in run.calls[1] + + +_API_DRIVER_CASES = case_params( + _api_driver_multi_turn_tool_loop_and_usage_accumulation, + _api_driver_refusal_records_calls_but_executes_nothing, + _api_driver_pairs_results_by_id_not_ordinal, + _api_driver_flags_result_id_mismatch, + _api_driver_iteration_cap_only_flags_mid_tool_loop, + _api_driver_clean_end_on_last_iteration_is_not_capped, + _api_driver_uses_optional_backend_token_counter, + _api_driver_records_only_matching_evidence_labels, +) + + +@pytest.mark.parametrize("case", _API_DRIVER_CASES) +def test_api_driver_behaviours(case): + case() @pytest.mark.parametrize( @@ -498,183 +552,198 @@ def test_openai_backend_normalizes_and_preserves_stop_reason(raw_reason, expecte assert turn.provider_stop_reason == raw_reason -def test_openai_backend_behaviours(): - def test_openai_backend_translates_tools_calls_and_tool_messages(): - responses = [ +def _openai_backend_translates_tools_calls_and_tool_messages(): + responses = [ + { + "model": "gpt-actual", + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + } + ], + }, + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 3, + "prompt_tokens_details": {"cached_tokens": 5}, + }, + }, + { + "model": "gpt-actual", + "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], + "usage": {"prompt_tokens": 20, "completion_tokens": 4}, + }, + ] + completions = FakeOpenAICompletions(responses) + client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) + backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("call-1", "value")]) + second = backend.next_turn() + + first_request = completions.requests[0] + assert first_request["max_completion_tokens"] == 321 + assert first_request["messages"] == [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "prompt"}, + ] + assert first_request["tools"] == [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + } + ] + assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] + assert first.stop_reason is StopReason.TOOL_USE + assert first.provider_stop_reason == "tool_calls" + assert first.usage == Usage(12, 3, 5, 0) + second_messages = completions.requests[1]["messages"] + assert second_messages[2] == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + } + ], + } + assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} + assert second.text == "done" + assert second.stop_reason is StopReason.END_TURN + assert second.provider_stop_reason == "stop" + assert backend.actual_model == "gpt-actual" + + +def _openai_backend_normalizes_refusal_for_driver_guard(): + completions = FakeOpenAICompletions( + [ { - "model": "gpt-actual", + "model": "gpt", "choices": [ { - "finish_reason": "tool_calls", + "finish_reason": "content_filter", "message": { "content": None, + "refusal": "declined", "tool_calls": [ { - "id": "call-1", + "id": "danger", "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + "function": {"name": "write", "arguments": "{}"}, } ], }, } ], - "usage": { - "prompt_tokens": 12, - "completion_tokens": 3, - "prompt_tokens_details": {"cached_tokens": 5}, - }, - }, - { - "model": "gpt-actual", - "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], - "usage": {"prompt_tokens": 20, "completion_tokens": 4}, - }, - ] - completions = FakeOpenAICompletions(responses) - client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) - backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) - tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) - - backend.start("system", "prompt", [tool]) - first = backend.next_turn() - backend.add_tool_results([ToolResult("call-1", "value")]) - second = backend.next_turn() - - first_request = completions.requests[0] - assert first_request["max_completion_tokens"] == 321 - assert first_request["messages"] == [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "prompt"}, - ] - assert first_request["tools"] == [ - { - "type": "function", - "function": { - "name": "lookup", - "description": "Look up", - "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, - }, + "usage": None, } ] - assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] - assert first.stop_reason is StopReason.TOOL_USE - assert first.provider_stop_reason == "tool_calls" - assert first.usage == Usage(12, 3, 5, 0) - second_messages = completions.requests[1]["messages"] - assert second_messages[2] == { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, - } - ], - } - assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} - assert second.text == "done" - assert second.stop_reason is StopReason.END_TURN - assert second.provider_stop_reason == "stop" - assert backend.actual_model == "gpt-actual" - - def test_openai_backend_normalizes_refusal_for_driver_guard(): - completions = FakeOpenAICompletions( - [ - { - "model": "gpt", - "choices": [ - { - "finish_reason": "content_filter", - "message": { - "content": None, - "refusal": "declined", - "tool_calls": [ - { - "id": "danger", - "type": "function", - "function": {"name": "write", "arguments": "{}"}, - } - ], - }, - } - ], - "usage": None, - } - ] - ) - backend = OpenAIBackend( - "gpt", - max_tokens=10, - client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), - ) - backend.start(None, "prompt", []) + ) + backend = OpenAIBackend( + "gpt", + max_tokens=10, + client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), + ) + backend.start(None, "prompt", []) - turn = backend.next_turn() + turn = backend.next_turn() - assert turn.stop_reason is StopReason.REFUSAL - assert turn.provider_stop_reason == "content_filter" - assert turn.text == "declined" - assert turn.tool_calls == [ToolCall("danger", "write", {})] + assert turn.stop_reason is StopReason.REFUSAL + assert turn.provider_stop_reason == "content_filter" + assert turn.text == "declined" + assert turn.tool_calls == [ToolCall("danger", "write", {})] - test_openai_backend_translates_tools_calls_and_tool_messages() - test_openai_backend_normalizes_refusal_for_driver_guard() +@pytest.mark.parametrize( + "case", + case_params( + _openai_backend_translates_tools_calls_and_tool_messages, + _openai_backend_normalizes_refusal_for_driver_guard, + ), +) +def test_openai_backend_behaviours(case): + case() -def test_tool_behaviours(): - def test_tool_spec_from_mcp_reads_dict_and_object_entries(): - from evals.drivers.driver import tool_spec_from_mcp - as_dict = tool_spec_from_mcp( - {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} - ) - assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") - assert as_dict.input_schema == {"type": "object", "x": 1} - - as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) - assert as_object.name == "create_cycle" - # A missing or non-dict schema must still yield a usable object schema. - assert as_object.input_schema == {"type": "object"} - assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} - - def test_tool_result_behaviours(): - def test_tool_result_from_mcp_text_only_joins_blocks(): - from evals.drivers.driver import tool_result_from_mcp - - result = tool_result_from_mcp( - "call_1", - {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, - ) - assert (result.call_id, result.text, result.kind, result.is_error) == ( - "call_1", - "first\nsecond", - "text", - False, - ) +def _tool_spec_from_mcp_reads_dict_and_object_entries(): + from evals.drivers.driver import tool_spec_from_mcp - def test_tool_result_from_mcp_serializes_non_text_blocks(): - from evals.drivers.driver import tool_result_from_mcp + as_dict = tool_spec_from_mcp( + {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} + ) + assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") + assert as_dict.input_schema == {"type": "object", "x": 1} + + as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) + assert as_object.name == "create_cycle" + # A missing or non-dict schema must still yield a usable object schema. + assert as_object.input_schema == {"type": "object"} + assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} - mixed = tool_result_from_mcp( - "call_2", - {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, - ) - assert mixed.kind == "mixed" - assert '"image"' in mixed.text and "chart:" in mixed.text - image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) - assert image_only.kind == "image" - assert '"data":"AAAA"' in image_only.text +def _tool_result_from_mcp_text_only_joins_blocks(): + from evals.drivers.driver import tool_result_from_mcp - def test_tool_result_from_mcp_propagates_error_flag_in_both_spellings(): - from evals.drivers.driver import tool_result_from_mcp + result = tool_result_from_mcp( + "call_1", + {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + ) + assert (result.call_id, result.text, result.kind, result.is_error) == ( + "call_1", + "first\nsecond", + "text", + False, + ) - assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True - assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True - test_tool_result_from_mcp_text_only_joins_blocks() - test_tool_result_from_mcp_serializes_non_text_blocks() - test_tool_result_from_mcp_propagates_error_flag_in_both_spellings() +def _tool_result_from_mcp_serializes_non_text_blocks(): + from evals.drivers.driver import tool_result_from_mcp - test_tool_spec_from_mcp_reads_dict_and_object_entries() - test_tool_result_behaviours() + mixed = tool_result_from_mcp( + "call_2", + {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, + ) + assert mixed.kind == "mixed" + assert '"image"' in mixed.text and "chart:" in mixed.text + + image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) + assert image_only.kind == "image" + assert '"data":"AAAA"' in image_only.text + + +def _tool_result_from_mcp_propagates_error_flag_in_both_spellings(): + from evals.drivers.driver import tool_result_from_mcp + + assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True + assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True + + +@pytest.mark.parametrize( + "case", + case_params( + _tool_spec_from_mcp_reads_dict_and_object_entries, + _tool_result_from_mcp_text_only_joins_blocks, + _tool_result_from_mcp_serializes_non_text_blocks, + _tool_result_from_mcp_propagates_error_flag_in_both_spellings, + ), +) +def test_tool_behaviours(case): + case() diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 173ceda5..24c234c4 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -27,6 +27,8 @@ wait_for_proxy_meta, ) from evals.drivers.driver import CliDriver, CliLaunch, CliOutput +from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE +from tests.evals.conftest import case_params def _pid_alive(pid: int) -> bool: @@ -42,13 +44,12 @@ def _pid_alive(pid: int) -> bool: REPO = Path(__file__).resolve().parents[3] -def test_run_behaviours(tmp_path, monkeypatch): - def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path): - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky_cli.py" - script.write_text( - textwrap.dedent( - f""" +def _run_cli_subprocess_kills_process_group_on_timeout(tmp_path, _monkeypatch): + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_cli.py" + script.write_text( + textwrap.dedent( + f""" import os, subprocess, sys, time from pathlib import Path pidfile = Path({str(pidfile)!r}) @@ -60,43 +61,44 @@ def test_run_cli_subprocess_kills_process_group_on_timeout(tmp_path): # Hold our stdout open forever (simulates grandchild pipe hold). time.sleep(9999) """ - ), - encoding="utf-8", - ) + ), + encoding="utf-8", + ) - t0 = time.monotonic() - with pytest.raises(subprocess.TimeoutExpired) as ei: - run_cli_subprocess( - [sys.executable, str(script)], - timeout=1.0, - capture_output=True, - text=True, - ) - elapsed = time.monotonic() - t0 - assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" - assert getattr(ei.value, "killed_process_group", False) is True + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired) as ei: + run_cli_subprocess( + [sys.executable, str(script)], + timeout=1.0, + capture_output=True, + text=True, + ) + elapsed = time.monotonic() - t0 + assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" + assert getattr(ei.value, "killed_process_group", False) is True + + # Wait briefly for reaping + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"process group members still alive: {alive}" + + +def _run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): + import evals.drivers as drivers_mod - # Wait briefly for reaping - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): - break - time.sleep(0.05) - assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"process group members still alive: {alive}" - - def test_run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): - import evals.drivers as drivers_mod - - pidfile = tmp_path / "pids.txt" - script = tmp_path / "sticky.py" - script.write_text( - textwrap.dedent( - f""" + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky.py" + script.write_text( + textwrap.dedent( + f""" import os, subprocess, sys, time from pathlib import Path pidfile = Path({str(pidfile)!r}) @@ -104,58 +106,58 @@ def test_run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") time.sleep(9999) """ - ), - encoding="utf-8", - ) - - real_comm = subprocess.Popen.communicate - calls = {"n": 0} - - def boom_communicate(self, *a, **k): - calls["n"] += 1 - if calls["n"] == 1: - # Wait until pidfile is written so we can assert both die. - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: - break - time.sleep(0.02) - raise KeyboardInterrupt("injected mid-communicate") - return real_comm(self, *a, **k) - - monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + ), + encoding="utf-8", + ) - t0 = time.monotonic() - with pytest.raises(KeyboardInterrupt): - run_cli_subprocess( - [sys.executable, str(script)], - timeout=30.0, - capture_output=True, - text=True, - ) - assert time.monotonic() - t0 < 6.0 + real_comm = subprocess.Popen.communicate + calls = {"n": 0} - deadline = time.monotonic() + 3.0 - pids: list[int] = [] - while time.monotonic() < deadline: - if pidfile.is_file(): - pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] - if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + def boom_communicate(self, *a, **k): + calls["n"] += 1 + if calls["n"] == 1: + # Wait until pidfile is written so we can assert both die. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: break - time.sleep(0.05) - assert len(pids) == 2 - alive = [p for p in pids if _pid_alive(p)] - assert not alive, f"group survived BaseException path: {alive}" - # silence unused import lint if any - assert drivers_mod.run_cli_subprocess is run_cli_subprocess - - _d0 = tmp_path / "test_run_cli_subprocess_kills_process_group_on_timeout" - _d0.mkdir() - test_run_cli_subprocess_kills_process_group_on_timeout(_d0) - _d1 = tmp_path / "test_run_cli_subprocess_baseexception_kills_group" - _d1.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_run_cli_subprocess_baseexception_kills_group(_d1, mp) + time.sleep(0.02) + raise KeyboardInterrupt("injected mid-communicate") + return real_comm(self, *a, **k) + + monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_cli_subprocess( + [sys.executable, str(script)], + timeout=30.0, + capture_output=True, + text=True, + ) + assert time.monotonic() - t0 < 6.0 + + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2 + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"group survived BaseException path: {alive}" + # silence unused import lint if any + assert drivers_mod.run_cli_subprocess is run_cli_subprocess + + +@pytest.mark.parametrize( + "case", + case_params(_run_cli_subprocess_kills_process_group_on_timeout, _run_cli_subprocess_baseexception_kills_group), +) +def test_run_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): @@ -234,141 +236,144 @@ def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): pass -def test_cli_behaviours(tmp_path, monkeypatch): - def test_cli_driver_timeout_notes_process_group_kill(tmp_path): - script = tmp_path / "slow.py" - script.write_text( - textwrap.dedent( - """ +def _cli_driver_timeout_notes_process_group_kill(tmp_path, _monkeypatch): + script = tmp_path / "slow.py" + script.write_text( + textwrap.dedent( + """ import time time.sleep(9999) """ - ), - encoding="utf-8", - ) - - # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. - from evals.drivers import run_cli_subprocess as real_runner - - def short_timeout_runner(cmd, **kwargs): - kwargs = dict(kwargs) - kwargs["timeout"] = 0.5 - # Replace the CLI binary with our sticky sleeper - return real_runner([sys.executable, str(script)], **kwargs) + ), + encoding="utf-8", + ) - driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) - t0 = time.monotonic() - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert time.monotonic() - t0 < 6.0 - assert run.stopped_reason == "timeout" - assert "timeout_killed_process_group" in run.notes - - def test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path, monkeypatch): - clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) - - class MinimalCliDriver(CliDriver): - name = "minimal-cli" - temp_dir_prefix = "plane-eval-minimal-" - - def write_mcp_config( - self, - temp_dir: Path, - *, - task_cwd: Path, - server_command: list[str], - child_env: dict[str, str], - ) -> CliLaunch: - del temp_dir, child_env - # Harness-owned setup takes five seconds on the fake clock. The - # persisted wall time must start after this hook returns. - clock["now"] = 5.0 - self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) - return CliLaunch(cwd=task_cwd) - - def build_command( - self, - prompt: str, - *, - model: str | None, - max_turns: int, - system: str | None, - launch: CliLaunch, - ) -> list[str]: - del model, max_turns, system, launch - return ["minimal", prompt] - - def parse_output( - self, - proc: subprocess.CompletedProcess[str], - *, - task_cwd: Path, - max_turns: int, - notes: list[str], - ) -> CliOutput: - del proc, task_cwd, max_turns, notes - return CliOutput( - final_text="done", - calls=[ - {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, - {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, - ], - ) + # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. + from evals.drivers import run_cli_subprocess as real_runner + + def short_timeout_runner(cmd, **kwargs): + kwargs = dict(kwargs) + kwargs["timeout"] = 0.5 + # Replace the CLI binary with our sticky sleeper + return real_runner([sys.executable, str(script)], **kwargs) + + driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) + t0 = time.monotonic() + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert time.monotonic() - t0 < 6.0 + assert run.stopped_reason == "timeout" + assert "timeout_killed_process_group" in run.notes + + +def _cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path, monkeypatch): + clock = {"now": 0.0} + monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) + + class MinimalCliDriver(CliDriver): + name = "minimal-cli" + temp_dir_prefix = "plane-eval-minimal-" + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir, child_env + # Harness-owned setup takes five seconds on the fake clock. The + # persisted wall time must start after this hook returns. + clock["now"] = 5.0 + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del model, max_turns, system, launch + return ["minimal", prompt] + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del proc, task_cwd, max_turns, notes + return CliOutput( + final_text="done", + calls=[ + {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, + {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, + ], + ) - def write_complete_sidecar(path: Path, tool: str) -> None: - rows = [ - { - "tool": tool, - "args": {}, - "is_error": False, - "result_chars": 2, - "duration_ms": 1, - "seq": 1, - }, - {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, - ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") - - success_driver: MinimalCliDriver - - def success_runner(cmd, **kwargs): - write_complete_sidecar(success_driver.sidecar_path, "proxy_first") - clock["now"] = 7.0 - return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") - - success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) - success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert success.call_source == "proxy" - assert [call["tool"] for call in success.calls] == ["proxy_first"] - assert success.wall_time_s == 2.0 - - timeout_driver: MinimalCliDriver - - def timeout_runner(cmd, **kwargs): - write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") - clock["now"] = 8.0 - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) - - timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) - timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) - assert timed_out.stopped_reason == "timeout" - assert timed_out.call_source == "proxy" - assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] - assert timed_out.wall_time_s == 3.0 - - _d0 = tmp_path / "test_cli_driver_timeout_notes_process_group_kill" - _d0.mkdir() - test_cli_driver_timeout_notes_process_group_kill(_d0) - _d1 = tmp_path / "test_cli_driver_template_inherits_proxy_first_and_timeout_harvest" - _d1.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_cli_driver_template_inherits_proxy_first_and_timeout_harvest(_d1, mp) + def write_complete_sidecar(path: Path, tool: str) -> None: + rows = [ + { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + }, + {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + success_driver: MinimalCliDriver + + def success_runner(cmd, **kwargs): + write_complete_sidecar(success_driver.sidecar_path, "proxy_first") + clock["now"] = 7.0 + return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") + + success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) + success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert success.call_source == "proxy" + assert [call["tool"] for call in success.calls] == ["proxy_first"] + assert success.wall_time_s == 2.0 + + timeout_driver: MinimalCliDriver + + def timeout_runner(cmd, **kwargs): + write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") + clock["now"] = 8.0 + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) + + timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) + timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert timed_out.stopped_reason == "timeout" + assert timed_out.call_source == "proxy" + assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] + assert timed_out.wall_time_s == 3.0 + + +@pytest.mark.parametrize( + "case", + case_params( + _cli_driver_timeout_notes_process_group_kill, + _cli_driver_template_inherits_proxy_first_and_timeout_harvest, + ), +) +def test_cli_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) def test_old_payload_free_sidecar_still_parses(tmp_path: Path): @@ -396,82 +401,117 @@ def test_old_payload_free_sidecar_still_parses(tmp_path: Path): assert "result_text" not in calls[0] -def test_apply_behaviours(tmp_path): - def test_apply_proxy_sidecar_replaces_when_nonempty(tmp_path): - side = tmp_path / "s.jsonl" - side.write_text( - json.dumps( - { - "tool": "find_work_items", - "args": {"q": "x"}, - "is_error": False, - "result_chars": 12, - "duration_ms": 5, - "seq": 1, - } - ) - + "\n", - encoding="utf-8", - ) - notes: list[str] = [] - calls, client, src = apply_proxy_sidecar( - [{"tool": "old", "args": {}, "origin": "plane"}], - [], - side, - notes, +def _apply_proxy_sidecar_replaces_when_nonempty(tmp_path): + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps( + { + "tool": "find_work_items", + "args": {"q": "x"}, + "is_error": False, + "result_chars": 12, + "duration_ms": 5, + "seq": 1, + } ) - assert src == "proxy" - assert calls[0]["tool"] == "find_work_items" - assert calls[0]["duration_ms"] == 5 - assert any("calls_from_proxy" in n for n in notes) - - def test_apply_proxy_sidecar_empty_fallback(tmp_path): - side = tmp_path / "empty.jsonl" - side.write_text("", encoding="utf-8") - notes: list[str] = [] - original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] - calls, _client, src = apply_proxy_sidecar(original, [], side, notes) - assert calls is original or calls == original - assert "proxy_sidecar_empty" in notes - assert src != "proxy" or calls == original - - def test_apply_proxy_incomplete_defers_to_richer_cli(tmp_path): - p = tmp_path / "s.jsonl" - # Incomplete: one proxy call, no meta. - p.write_text( - json.dumps( - { - "tool": "from_proxy", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - ) - + "\n", - encoding="utf-8", + + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, client, src = apply_proxy_sidecar( + [{"tool": "old", "args": {}, "origin": "plane"}], + [], + side, + notes, + ) + assert src == "proxy" + assert calls[0]["tool"] == "find_work_items" + assert calls[0]["duration_ms"] == 5 + assert any("calls_from_proxy" in n for n in notes) + + +def _apply_proxy_sidecar_empty_fallback(tmp_path): + side = tmp_path / "empty.jsonl" + side.write_text("", encoding="utf-8") + notes: list[str] = [] + original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] + calls, _client, src = apply_proxy_sidecar(original, [], side, notes) + assert calls is original or calls == original + assert "proxy_sidecar_empty" in notes + assert src != "proxy" or calls == original + + +def _apply_proxy_incomplete_defers_to_richer_cli(tmp_path): + p = tmp_path / "s.jsonl" + # Incomplete: one proxy call, no meta. + p.write_text( + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } ) - cli = [ - {"tool": "c1", "args": {}, "origin": "plane"}, - {"tool": "c2", "args": {}, "origin": "plane"}, - ] - notes: list[str] = [] - calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) - assert src != "proxy" - assert [c["tool"] for c in calls] == ["c1", "c2"] - assert any("proxy_sidecar_incomplete" in n for n in notes) - assert any("deferred_to_cli" in n for n in notes) - - _d0 = tmp_path / "test_apply_proxy_sidecar_replaces_when_nonempty" - _d0.mkdir() - test_apply_proxy_sidecar_replaces_when_nonempty(_d0) - _d1 = tmp_path / "test_apply_proxy_sidecar_empty_fallback" - _d1.mkdir() - test_apply_proxy_sidecar_empty_fallback(_d1) - _d2 = tmp_path / "test_apply_proxy_incomplete_defers_to_richer_cli" - _d2.mkdir() - test_apply_proxy_incomplete_defers_to_richer_cli(_d2) + + "\n", + encoding="utf-8", + ) + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in calls] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def _apply_proxy_with_skipped_row_defers_to_richer_cli(tmp_path): + p = tmp_path / "s.jsonl" + rows = [ + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + ), + "{corrupted mid-stream row", + json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}), + ] + p.write_text("\n".join(rows) + "\n", encoding="utf-8") + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + + assert src == "json" + assert [call["tool"] for call in calls] == ["c1", "c2"] + assert "proxy_sidecar_incomplete:skipped_rows=1" in notes + assert "proxy_sidecar_deferred_to_cli_trace" in notes + + +@pytest.mark.parametrize( + "case", + case_params( + _apply_proxy_sidecar_replaces_when_nonempty, + _apply_proxy_sidecar_empty_fallback, + _apply_proxy_incomplete_defers_to_richer_cli, + _apply_proxy_with_skipped_row_defers_to_richer_cli, + ), +) +def test_apply_behaviours(case, tmp_path): + case(tmp_path) def test_proxy_wrap_server_command(): @@ -493,51 +533,81 @@ def test_proxy_wrap_server_command(): assert with_payloads[5:7] == ["--record-result-payloads", "--"] -def test_load_behaviours(tmp_path): - def test_load_proxy_sidecar_sorts_by_seq(tmp_path): - p = tmp_path / "s.jsonl" - # Append in reverse response order. - rows = [ - {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, - {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, - { - "row_type": "proxy_meta", - "relayed_lines": 2, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - }, - ] - p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") - calls = load_proxy_sidecar_calls(p) - assert [c["tool"] for c in calls] == ["a", "b"] - - def test_load_proxy_sidecar_torn_final_line(tmp_path): - p = tmp_path / "s.jsonl" - good = { - "tool": "a", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - # Complete call row + torn final line (no proxy_meta). - p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") - calls, status = load_proxy_sidecar(p) - assert status["state"] == "incomplete" - assert status["torn_line"] is True - assert status["meta"] is None - assert [c["tool"] for c in calls] == ["a"] - - _d0 = tmp_path / "test_load_proxy_sidecar_sorts_by_seq" - _d0.mkdir() - test_load_proxy_sidecar_sorts_by_seq(_d0) - _d1 = tmp_path / "test_load_proxy_sidecar_torn_final_line" - _d1.mkdir() - test_load_proxy_sidecar_torn_final_line(_d1) +def _load_proxy_sidecar_sorts_by_seq(tmp_path): + p = tmp_path / "s.jsonl" + # Append in reverse response order. + rows = [ + {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, + {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls = load_proxy_sidecar_calls(p) + assert [c["tool"] for c in calls] == ["a", "b"] + + +def _load_proxy_sidecar_torn_final_line(tmp_path): + p = tmp_path / "s.jsonl" + good = { + "tool": "a", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + # Complete call row + torn final line (no proxy_meta). + p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status["torn_line"] is True + assert status["meta"] is None + assert [c["tool"] for c in calls] == ["a"] + + +@pytest.mark.parametrize( + "case", + case_params(_load_proxy_sidecar_sorts_by_seq, _load_proxy_sidecar_torn_final_line), +) +def test_load_behaviours(case, tmp_path): + case(tmp_path) + + +@pytest.mark.parametrize( + ("bad_row", "case_id"), + [ + ("{corrupted mid-stream row", "invalid-json"), + (json.dumps(["not", "an", "object"]), "non-object-json"), + (json.dumps({"args": {"lost": "tool"}}), "missing-tool"), + ], + ids=lambda value: value if value in {"invalid-json", "non-object-json", "missing-tool"} else None, +) +def test_load_proxy_sidecar_skipped_row_makes_trace_incomplete(tmp_path: Path, bad_row: str, case_id: str): + path = tmp_path / f"{case_id}.jsonl" + call = { + "tool": "visible", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False} + path.write_text("\n".join((json.dumps(call), bad_row, json.dumps(meta))) + "\n", encoding="utf-8") + + calls, status = load_proxy_sidecar(path) + + assert [row["tool"] for row in calls] == ["visible"] + assert status["skipped_rows"] == 1 + assert status["state"] == "incomplete" def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): @@ -639,66 +709,18 @@ def test_ensure_proxy_pythonpath_injects_repo(): assert env2["PYTHONPATH"].count(str(REPO)) == 1 -def test_timeout_behaviours(tmp_path): - def test_timeout_harvests_sidecar_calls(tmp_path): - side_calls = [ - { - "tool": "pre_timeout", - "args": {"a": 1}, - "is_error": False, - "result_chars": 3, - "duration_ms": 1, - "seq": 1, - }, - { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - }, - ] - - def fake_run(cmd, **kwargs): - # Plant a complete sidecar next to the mcp config (temp dir still alive). - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - # Sidecar path is in the same temp dir as mcp.json for Claude. - # Find sidecar from proxy args in mcp config. - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - log_idx = args.index("--log") + 1 - side = Path(args[log_idx]) - side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "pre_timeout" - - def test_timeout_harvest_waits_for_delayed_meta(tmp_path): - import threading - import time as time_mod - - call_row = { - "tool": "late_meta_tool", - "args": {"n": 1}, +def _timeout_harvests_sidecar_calls(tmp_path): + side_calls = [ + { + "tool": "pre_timeout", + "args": {"a": 1}, "is_error": False, - "result_chars": 2, + "result_chars": 3, "duration_ms": 1, "seq": 1, - } - meta_row = { + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + }, + { "row_type": "proxy_meta", "relayed_lines": 1, "unparsed_lines": 0, @@ -706,52 +728,155 @@ def test_timeout_harvest_waits_for_delayed_meta(tmp_path): "notifications": 0, "pending_left": 0, "child_killed": False, + "evidence_trace_available": True, + }, + ] + + def fake_run(cmd, **kwargs): + # Plant a complete sidecar next to the mcp config (temp dir still alive). + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + # Sidecar path is in the same temp dir as mcp.json for Claude. + # Find sidecar from proxy args in mcp config. + mcp = json.loads(cfg.read_text()) + assert EVIDENCE_SENTINELS_ENV in mcp["mcpServers"]["plane"]["env"] + args = mcp["mcpServers"]["plane"]["args"] + log_idx = args.index("--log") + 1 + side = Path(args[log_idx]) + side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "pre_timeout" + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.evidence_trace_available is True + + +def _timeout_harvest_waits_for_delayed_meta(tmp_path): + import threading + import time as time_mod + + call_row = { + "tool": "late_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + meta_row = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": False, + } + seen: dict = {"waited": False} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + # Call row first — no meta yet (simulates proxy still finalizing). + side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") + + def write_meta_later() -> None: + time_mod.sleep(0.45) + with side.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta_row) + "\n") + seen["waited"] = True + + threading.Thread(target=write_meta_later, daemon=True).start() + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + t0 = time_mod.monotonic() + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + elapsed = time_mod.monotonic() - t0 + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "late_meta_tool" + assert seen["waited"] is True + # Must have waited for the delayed meta (~0.45s), not returned instantly. + assert elapsed >= 0.4 + assert "proxy_meta_wait_timeout" not in run.notes + + +def _timeout_incomplete_sidecar_cannot_supply_response_evidence(tmp_path): + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + call = { + "tool": "evidence_call", + "args": {}, + "is_error": False, + "result_chars": 5, + "duration_ms": 1, + "seq": 1, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + meta = { + "row_type": "proxy_meta", + "pending_left": 0, "pumps_alive": False, + "evidence_trace_available": True, } - seen: dict = {"waited": False} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - mcp = json.loads(cfg.read_text()) - args = mcp["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - # Call row first — no meta yet (simulates proxy still finalizing). - side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") - - def write_meta_later() -> None: - time_mod.sleep(0.45) - with side.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(meta_row) + "\n") - seen["waited"] = True - - threading.Thread(target=write_meta_later, daemon=True).start() - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - t0 = time_mod.monotonic() - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, + side.write_text( + "\n".join((json.dumps(call), "{corrupted mid-stream row", json.dumps(meta))) + "\n", + encoding="utf-8", ) - elapsed = time_mod.monotonic() - t0 - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "late_meta_tool" - assert seen["waited"] is True - # Must have waited for the delayed meta (~0.45s), not returned instantly. - assert elapsed >= 0.4 - assert "proxy_meta_wait_timeout" not in run.notes - - _d0 = tmp_path / "test_timeout_harvests_sidecar_calls" - _d0.mkdir() - test_timeout_harvests_sidecar_calls(_d0) - _d1 = tmp_path / "test_timeout_harvest_waits_for_delayed_meta" - _d1.mkdir() - test_timeout_harvest_waits_for_delayed_meta(_d1) + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + ) + + assert run.call_source == "proxy" + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.evidence_trace_available is False + assert "proxy_sidecar_incomplete:skipped_rows=1" in run.notes + assert "proxy_response_evidence_unavailable" in run.notes + + +@pytest.mark.parametrize( + "case", + case_params( + _timeout_harvests_sidecar_calls, + _timeout_harvest_waits_for_delayed_meta, + _timeout_incomplete_sidecar_cannot_supply_response_evidence, + ), +) +def test_timeout_behaviours(case, tmp_path): + case(tmp_path) def test_wait_for_proxy_meta_unit(tmp_path: Path): diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index 0cb07fd6..fe3305da 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -30,6 +30,7 @@ from evals.tool_names import ( split_plane_and_client_calls, ) +from tests.evals.conftest import case_params CLAUDE_JSON_RESULT = { "type": "result", @@ -254,240 +255,247 @@ def test_normalize_claude_usage_real_shape(): assert total["source"] == "modelUsage" -def test_parse_behaviours(tmp_path): - def test_parse_claude_behaviours(): - def test_parse_claude_json_result_usage_and_cost(): - out = parse_claude_json_result(CLAUDE_JSON_RESULT) - assert out["final_text"] == "The work item is in Todo." - assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" - assert out["num_turns"] == 3 - assert out["usage"]["input_tokens"] == 10 - assert out["usage"]["total_cost_usd"] == 0.291 - assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 - assert out["calls"] == [] - assert out["stopped_reason"] == "end_turn" - - def test_parse_claude_json_with_embedded_calls_splits_toolsearch(): - out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) - assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] - assert all(c["origin"] == "plane" for c in out["calls"]) - assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] - assert out["calls"][0]["args"]["limit"] == 10 - - def test_parse_claude_json_preserves_error_subtype(): - out = parse_claude_json_result( +def _parse_claude_json_result_usage_and_cost(_tmp_path): + out = parse_claude_json_result(CLAUDE_JSON_RESULT) + assert out["final_text"] == "The work item is in Todo." + assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + assert out["num_turns"] == 3 + assert out["usage"]["input_tokens"] == 10 + assert out["usage"]["total_cost_usd"] == 0.291 + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["calls"] == [] + assert out["stopped_reason"] == "end_turn" + + +def _parse_claude_json_with_embedded_calls_splits_toolsearch(_tmp_path): + out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) + assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] + assert all(c["origin"] == "plane" for c in out["calls"]) + assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] + assert out["calls"][0]["args"]["limit"] == 10 + + +def _parse_claude_json_preserves_error_subtype(_tmp_path): + out = parse_claude_json_result( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "x", + "session_id": "s", + "num_turns": 1, + } + ) + assert out["stopped_reason"] == "error_during_execution" + + +def _parse_claude_transcript_calls(tmp_path): + p = tmp_path / "sess.jsonl" + p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + tagged = parse_claude_transcript_calls(p) + plane, client = split_plane_and_client_calls(tagged) + assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in client] == ["ToolSearch"] + assert plane[0]["args"]["project_id"] == "proj-1" + + +def _parse_codex_jsonl_events(_tmp_path): + out = parse_codex_jsonl_events(CODEX_JSONL) + assert out["session_id"] == "sess-codex-1" + assert out["final_text"] == "All set." + assert out["usage"]["input_tokens"] == 5000 + assert out["usage"]["cache_read_input_tokens"] == 1000 + # plane only in calls; exec_command is client machinery + tools = [c["tool"] for c in out["calls"]] + assert tools == ["find_work_items"] + assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] + assert out["stopped_reason"] == "end_turn" + + +def _parse_codex_jsonl_events_v0147_schema(_tmp_path): + out = parse_codex_jsonl_events(CODEX_V0147_JSONL) + assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" + assert out["final_text"] == "PING" + assert out["usage"]["input_tokens"] == 16050 + assert out["usage"]["cache_read_input_tokens"] == 15104 + assert out["usage"]["cache_creation_input_tokens"] == 0 + assert out["usage"]["output_tokens"] == 5 + + +def _parse_codex_jsonl_events_mixed_old_and_new_schema(_tmp_path): + mixed = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), + json.dumps( { - "type": "result", - "subtype": "error_during_execution", - "is_error": True, - "result": "x", - "session_id": "s", - "num_turns": 1, + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, } - ) - assert out["stopped_reason"] == "error_during_execution" - - test_parse_claude_json_result_usage_and_cost() - test_parse_claude_json_with_embedded_calls_splits_toolsearch() - test_parse_claude_json_preserves_error_subtype() - - def test_parse_claude_transcript_calls(tmp_path): - p = tmp_path / "sess.jsonl" - p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - tagged = parse_claude_transcript_calls(p) - plane, client = split_plane_and_client_calls(tagged) - assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] - assert [c["tool"] for c in client] == ["ToolSearch"] - assert plane[0]["args"]["project_id"] == "proj-1" - - def test_parse_codex_behaviours(): - def test_parse_codex_jsonl_events(): - out = parse_codex_jsonl_events(CODEX_JSONL) - assert out["session_id"] == "sess-codex-1" - assert out["final_text"] == "All set." - assert out["usage"]["input_tokens"] == 5000 - assert out["usage"]["cache_read_input_tokens"] == 1000 - # plane only in calls; exec_command is client machinery - tools = [c["tool"] for c in out["calls"]] - assert tools == ["find_work_items"] - assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] - assert out["stopped_reason"] == "end_turn" - - def test_parse_codex_jsonl_events_v0147_schema(): - out = parse_codex_jsonl_events(CODEX_V0147_JSONL) - assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" - assert out["final_text"] == "PING" - assert out["usage"]["input_tokens"] == 16050 - assert out["usage"]["cache_read_input_tokens"] == 15104 - assert out["usage"]["cache_creation_input_tokens"] == 0 - assert out["usage"]["output_tokens"] == 5 - - def test_parse_codex_jsonl_events_mixed_old_and_new_schema(): - mixed = "\n".join( - [ - json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), - json.dumps( - { - "type": "item.completed", - "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, - } - ), - # Legacy call row still harvested - json.dumps( - { - "type": "response_item", - "payload": { - "type": "function_call", - "name": "mcp__plane__list_work_items", - "arguments": json.dumps({"project_id": "p"}), - }, - } - ), - json.dumps( - { - "type": "turn.completed", - "usage": { - "input_tokens": 10, - "cached_input_tokens": 0, - "cache_write_input_tokens": 0, - "output_tokens": 2, - }, - } - ), - ] - ) - out = parse_codex_jsonl_events(mixed) - assert out["session_id"] == "thread-new-1" - assert "Hello from new" in out["final_text"] - assert [c["tool"] for c in out["calls"]] == ["list_work_items"] - assert out["usage"]["input_tokens"] == 10 - - test_parse_codex_jsonl_events() - test_parse_codex_jsonl_events_v0147_schema() - test_parse_codex_jsonl_events_mixed_old_and_new_schema() - - test_parse_claude_behaviours() - _d1 = tmp_path / "test_parse_claude_transcript_calls" - _d1.mkdir() - test_parse_claude_transcript_calls(_d1) - test_parse_codex_behaviours() - - -def test_find_behaviours(tmp_path, monkeypatch): - def test_find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): - from evals import drivers as drivers_mod - - sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" - sessions.mkdir(parents=True) - tid = "019ff6af-69df-7022-b353-322ffe1ececb" - # Unrelated newer session (must never be returned when looking for tid) - other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" - other.write_text( - json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", - encoding="utf-8", - ) - # Exact match via filename suffix - match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" - match.write_text( - json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", - encoding="utf-8", - ) + ), + # Legacy call row still harvested + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__list_work_items", + "arguments": json.dumps({"project_id": "p"}), + }, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + }, + } + ), + ] + ) + out = parse_codex_jsonl_events(mixed) + assert out["session_id"] == "thread-new-1" + assert "Hello from new" in out["final_text"] + assert [c["tool"] for c in out["calls"]] == ["list_work_items"] + assert out["usage"]["input_tokens"] == 10 + + +@pytest.mark.parametrize( + "case", + case_params( + _parse_claude_json_result_usage_and_cost, + _parse_claude_json_with_embedded_calls_splits_toolsearch, + _parse_claude_json_preserves_error_subtype, + _parse_claude_transcript_calls, + _parse_codex_jsonl_events, + _parse_codex_jsonl_events_v0147_schema, + _parse_codex_jsonl_events_mixed_old_and_new_schema, + ), +) +def test_parse_behaviours(case, tmp_path): + case(tmp_path) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout(tid) - assert found is not None - assert tid in found.name - # Must not return the other concurrent session - assert "other-session" not in found.name - - assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None - assert drivers_mod.find_codex_rollout(None) is None - - def test_find_codex_rollout_session_meta_id(tmp_path, monkeypatch): - from evals import drivers as drivers_mod - - sessions = tmp_path / ".codex" / "sessions" - sessions.mkdir(parents=True) - p = sessions / "rollout-meta-only.jsonl" - p.write_text( - json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", - encoding="utf-8", - ) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout("sess-meta-42") - assert found is not None - assert found.name == "rollout-meta-only.jsonl" - - _d0 = tmp_path / "test_find_codex_rollout_exact_match_and_unmatched" - _d0.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_find_codex_rollout_exact_match_and_unmatched(_d0, mp) - _d1 = tmp_path / "test_find_codex_rollout_session_meta_id" - _d1.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_find_codex_rollout_session_meta_id(_d1, mp) - - -def test_codex_behaviours(tmp_path, monkeypatch): - def test_codex_driver_notes_rollout_unmatched_when_no_file(tmp_path, monkeypatch): - (tmp_path / ".codex" / "sessions").mkdir(parents=True) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") - - driver = CodexCliDriver(runner=fake_run, use_proxy=False) - run = driver.run_task( - "ping", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - # Final text still from stdout (new schema) — never from a wrong rollout - assert run.final_text == "PING" - # Unmatched note only when looking for enrichment; with final_text present - # need_rollout is false for final_text — still may note if no calls. - # v0147 fixture has no tool calls → need_rollout True → unmatched note. - assert "codex_rollout_unmatched" in run.notes - - def test_codex_driver_behaviours(): - def test_codex_driver_parses_fake_stdout_no_live(): - def fake_run(cmd, **kwargs): - assert cmd[0] == "codex" - assert "exec" in cmd - assert "--json" in cmd - return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") - - driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed - run = driver.run_task( - "do it", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="gpt-test", - max_turns=5, - cwd=Path("/tmp"), - ) - assert run.experimental is True - assert run.call_source == "stream" - assert run.calls[0]["tool"] == "find_work_items" - assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] - assert run.usage is not None - assert run.usage["input_tokens"] == 5000 - assert run.final_text == "All set." - - def test_codex_driver_refuses_live_by_default(): - driver = CodexCliDriver() # real subprocess.run - with pytest.raises(RuntimeError, match="refuses live"): - driver.run_task("x", mcp_env={}, model=None, max_turns=1) - - test_codex_driver_parses_fake_stdout_no_live() - test_codex_driver_refuses_live_by_default() - - _d0 = tmp_path / "test_codex_driver_notes_rollout_unmatched_when_no_file" - _d0.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_codex_driver_notes_rollout_unmatched_when_no_file(_d0, mp) - test_codex_driver_behaviours() + +def _find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" + sessions.mkdir(parents=True) + tid = "019ff6af-69df-7022-b353-322ffe1ececb" + # Unrelated newer session (must never be returned when looking for tid) + other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" + other.write_text( + json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", + encoding="utf-8", + ) + # Exact match via filename suffix + match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" + match.write_text( + json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", + encoding="utf-8", + ) + + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout(tid) + assert found is not None + assert tid in found.name + # Must not return the other concurrent session + assert "other-session" not in found.name + + assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None + assert drivers_mod.find_codex_rollout(None) is None + + +def _find_codex_rollout_session_meta_id(tmp_path, monkeypatch): + from evals import drivers as drivers_mod + + sessions = tmp_path / ".codex" / "sessions" + sessions.mkdir(parents=True) + p = sessions / "rollout-meta-only.jsonl" + p.write_text( + json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = drivers_mod.find_codex_rollout("sess-meta-42") + assert found is not None + assert found.name == "rollout-meta-only.jsonl" + + +@pytest.mark.parametrize( + "case", + case_params(_find_codex_rollout_exact_match_and_unmatched, _find_codex_rollout_session_meta_id), +) +def test_find_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def _codex_driver_notes_rollout_unmatched_when_no_file(tmp_path, monkeypatch): + (tmp_path / ".codex" / "sessions").mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run, use_proxy=False) + run = driver.run_task( + "ping", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + # Final text still from stdout (new schema) — never from a wrong rollout + assert run.final_text == "PING" + # Unmatched note only when looking for enrichment; with final_text present + # need_rollout is false for final_text — still may note if no calls. + # v0147 fixture has no tool calls → need_rollout True → unmatched note. + assert "codex_rollout_unmatched" in run.notes + + +def _codex_driver_parses_fake_stdout_no_live(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + assert cmd[0] == "codex" + assert "exec" in cmd + assert "--json" in cmd + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="gpt-test", + max_turns=5, + cwd=Path("/tmp"), + ) + assert run.experimental is True + assert run.call_source == "stream" + assert run.calls[0]["tool"] == "find_work_items" + assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] + assert run.usage is not None + assert run.usage["input_tokens"] == 5000 + assert run.final_text == "All set." + + +def _codex_driver_refuses_live_by_default(_tmp_path, _monkeypatch): + driver = CodexCliDriver() # real subprocess.run + with pytest.raises(RuntimeError, match="refuses live"): + driver.run_task("x", mcp_env={}, model=None, max_turns=1) + + +@pytest.mark.parametrize( + "case", + case_params( + _codex_driver_notes_rollout_unmatched_when_no_file, + _codex_driver_parses_fake_stdout_no_live, + _codex_driver_refuses_live_by_default, + ), +) +def test_codex_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) def test_max_turns_detection_from_num_turns(): @@ -515,305 +523,308 @@ def fake_run(cmd, **kwargs): assert run.usage_total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 -def test_claude_behaviours(tmp_path, monkeypatch): - def test_claude_driver_falls_back_to_transcript(tmp_path, monkeypatch): - session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" - payload = { - **CLAUDE_JSON_RESULT, - "session_id": session_id, - "tool_calls": [], # force transcript path - "result": "from-json", - } +def _claude_driver_falls_back_to_transcript(tmp_path, monkeypatch): + session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], # force transcript path + "result": "from-json", + } - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") - # Plant transcript where find_claude_transcript looks - munged = str(tmp_path.resolve()).replace("/", "-") - proj = Path.home() / ".claude" / "projects" / munged - proj.mkdir(parents=True, exist_ok=True) - transcript = proj / f"{session_id}.jsonl" - transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - monkeypatch.setenv("HOME", str(Path.home())) # keep real home for this test path + # Keep transcript discovery isolated from the developer's real home. + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + munged = str(tmp_path.resolve()).replace("/", "-") + proj = Path.home() / ".claude" / "projects" / munged + proj.mkdir(parents=True, exist_ok=True) + transcript = proj / f"{session_id}.jsonl" + transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - driver = ClaudeCliDriver(runner=fake_run) - run = driver.run_task( - "prompt", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=10, - cwd=tmp_path, - ) - assert run.call_source == "transcript" - assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] - assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] - assert run.final_text == "from-json" - # cleanup planted file - transcript.unlink(missing_ok=True) - - def test_claude_driver_writes_mcp_config_and_cmd_flags(tmp_path): - seen: dict[str, Any] = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - seen["cwd"] = kwargs.get("cwd") - # Return minimal valid JSON - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), - stderr="", - ) - - driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") - driver.run_task( - "hello", - mcp_env={ - "PLANE_API_KEY": "key", - "PLANE_WORKSPACE_SLUG": "slug", - "PLANE_BASE_URL": "https://api.example", - "CUSTOM_SETTING": "enabled", - "PATH": "/usr/bin", - }, - model="sonnet", - max_turns=7, - cwd=tmp_path, - system="sys", - ) - cmd = seen["cmd"] - assert cmd[0] == "claude" - assert "-p" in cmd - assert "--output-format" in cmd and "json" in cmd - assert "--mcp-config" in cmd - assert "--max-turns" in cmd and "7" in cmd - assert "--model" in cmd and "sonnet" in cmd - assert "--permission-mode" in cmd and "bypassPermissions" in cmd - assert "--strict-mcp-config" in cmd - # mcp-config path is a temp file cleaned after run — re-check via write helper - cfg = tmp_path / "mcp.json" - write_claude_mcp_config( - cfg, - command="/venv/bin/python", - args=["-m", "plane_mcp", "stdio"], - env={"PLANE_API_KEY": "key"}, - ) - data = json.loads(cfg.read_text()) - assert "mcpServers" in data - assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] - - def test_claude_driver_server_command_override(tmp_path): - seen: dict[str, Any] = {} - - def fake_run(cmd, **kwargs): - # Capture the mcp.json content while it still exists (temp dir). - cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp_cfg"] = json.loads(cfg_path.read_text()) - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), - stderr="", - ) - - driver = ClaudeCliDriver( - runner=fake_run, - server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], - ) - driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, - model="sonnet", - max_turns=3, - cwd=tmp_path, + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, + ) + assert run.call_source == "transcript" + assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] + assert run.final_text == "from-json" + # cleanup planted file + transcript.unlink(missing_ok=True) + + +def _claude_driver_writes_mcp_config_and_cmd_flags(tmp_path, _monkeypatch): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + seen["cwd"] = kwargs.get("cwd") + # Return minimal valid JSON + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", ) - server = seen["mcp_cfg"]["mcpServers"]["plane"] - # Default use_proxy=True: command is the proxy; real server follows "--". - assert server["args"][:3] == ["-m", "evals.proxy", "--log"] - assert "--" in server["args"] - dash = server["args"].index("--") - assert server["args"][dash + 1 :] == [ - "/elsewhere/.venv/bin/plane-mcp-server", - "stdio", - "--mode", - "candidate", - ] - # Explicit foreign selection variables pass through to the child. - assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" - - def test_claude_driver_behaviours(): - def test_claude_driver_timeout_returns_agent_run_not_raise(): - def fake_run(cmd, **kwargs): - raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) - - driver = ClaudeCliDriver(runner=fake_run) - run = driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=2, - cwd=Path("/tmp"), - ) - assert run.stopped_reason == "timeout" - assert run.calls == [] - assert any("timeout after" in n for n in run.notes) - - def test_claude_driver_json_parse_failure_raises_for_infra_cli(): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") - - driver = ClaudeCliDriver(runner=fake_run) - with pytest.raises(RuntimeError, match="claude cli failed"): - driver.run_task( - "hello", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=Path("/tmp"), - ) - - test_claude_driver_timeout_returns_agent_run_not_raise() - test_claude_driver_json_parse_failure_raises_for_infra_cli() - - def test_claude_driver_uses_proxy_in_mcp_config(tmp_path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - # Leave empty sidecar (proxy not really run under fake runner). - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "done", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=3, - cwd=tmp_path, + driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") + driver.run_task( + "hello", + mcp_env={ + "PLANE_API_KEY": "key", + "PLANE_WORKSPACE_SLUG": "slug", + "PLANE_BASE_URL": "https://api.example", + "CUSTOM_SETTING": "enabled", + "PATH": "/usr/bin", + }, + model="sonnet", + max_turns=7, + cwd=tmp_path, + system="sys", + ) + cmd = seen["cmd"] + assert cmd[0] == "claude" + assert "-p" in cmd + assert "--output-format" in cmd and "json" in cmd + assert "--mcp-config" in cmd + assert "--max-turns" in cmd and "7" in cmd + assert "--model" in cmd and "sonnet" in cmd + assert "--permission-mode" in cmd and "bypassPermissions" in cmd + assert "--strict-mcp-config" in cmd + # mcp-config path is a temp file cleaned after run — re-check via write helper + cfg = tmp_path / "mcp.json" + write_claude_mcp_config( + cfg, + command="/venv/bin/python", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "key"}, + ) + data = json.loads(cfg.read_text()) + assert "mcpServers" in data + assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] + + +def _claude_driver_server_command_override(tmp_path, _monkeypatch): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + # Capture the mcp.json content while it still exists (temp dir). + cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp_cfg"] = json.loads(cfg_path.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["command"] == "/venv/bin/python" - assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] - assert "--" in server["args"] - assert "plane_mcp" in server["args"] - assert "proxy_sidecar_empty" in run.notes - - def test_claude_driver_proxy_disabled_no_wrap(tmp_path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["mcp"] = json.loads(cfg.read_text()) - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver = ClaudeCliDriver( + runner=fake_run, + server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], + ) + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp_cfg"]["mcpServers"]["plane"] + # Default use_proxy=True: command is the proxy; real server follows "--". + assert server["args"][:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + dash = server["args"].index("--") + assert server["args"][dash + 1 :] == [ + "/elsewhere/.venv/bin/plane-mcp-server", + "stdio", + "--mode", + "candidate", + ] + # Explicit foreign selection variables pass through to the child. + assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" + + +def _claude_driver_timeout_returns_agent_run_not_raise(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=2, + cwd=Path("/tmp"), + ) + assert run.stopped_reason == "timeout" + assert run.calls == [] + assert any("timeout after" in n for n in run.notes) + + +def _claude_driver_json_parse_failure_raises_for_infra_cli(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") + + driver = ClaudeCliDriver(runner=fake_run) + with pytest.raises(RuntimeError, match="claude cli failed"): driver.run_task( - "hi", + "hello", mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, model=None, max_turns=1, - cwd=tmp_path, + cwd=Path("/tmp"), ) - server = seen["mcp"]["mcpServers"]["plane"] - assert server["args"] == ["-m", "plane_mcp", "stdio"] - - def test_claude_mcp_env_has_pythonpath_when_proxied(tmp_path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - cfg = Path(cmd[cmd.index("--mcp-config") + 1]) - seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] - return subprocess.CompletedProcess( - cmd, - 0, - stdout=json.dumps( - { - "type": "result", - "subtype": "success", - "is_error": False, - "result": "ok", - "session_id": "s", - "num_turns": 1, - "usage": {"input_tokens": 1, "output_tokens": 1}, - } - ), - stderr="", - ) - ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model="sonnet", - max_turns=1, - cwd=tmp_path, + +def _claude_driver_uses_proxy_in_mcp_config(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + # Leave empty sidecar (proxy not really run under fake runner). + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", ) - assert str(REPO) in seen["env"].get("PYTHONPATH", "") - - _d0 = tmp_path / "test_claude_driver_falls_back_to_transcript" - _d0.mkdir() - with pytest.MonkeyPatch.context() as mp: - test_claude_driver_falls_back_to_transcript(_d0, mp) - _d1 = tmp_path / "test_claude_driver_writes_mcp_config_and_cmd_flags" - _d1.mkdir() - test_claude_driver_writes_mcp_config_and_cmd_flags(_d1) - _d2 = tmp_path / "test_claude_driver_server_command_override" - _d2.mkdir() - test_claude_driver_server_command_override(_d2) - test_claude_driver_behaviours() - _d4 = tmp_path / "test_claude_driver_uses_proxy_in_mcp_config" - _d4.mkdir() - test_claude_driver_uses_proxy_in_mcp_config(_d4) - _d5 = tmp_path / "test_claude_driver_proxy_disabled_no_wrap" - _d5.mkdir() - test_claude_driver_proxy_disabled_no_wrap(_d5) - _d6 = tmp_path / "test_claude_mcp_env_has_pythonpath_when_proxied" - _d6.mkdir() - test_claude_mcp_env_has_pythonpath_when_proxied(_d6) - - -def test_known_drivers_behaviours(): - def test_known_drivers(): - assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} - - def test_known_drivers_and_get_driver(): - assert "antigravity-cli" in KNOWN_DRIVERS - assert "opencode-cli" in KNOWN_DRIVERS - assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) - assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) - - test_known_drivers() - test_known_drivers_and_get_driver() + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["command"] == "/venv/bin/python" + assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + assert "plane_mcp" in server["args"] + assert "proxy_sidecar_empty" in run.notes + + +def _claude_driver_proxy_disabled_no_wrap(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["args"] == ["-m", "plane_mcp", "stdio"] + + +def _claude_mcp_env_has_pythonpath_when_proxied(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert str(REPO) in seen["env"].get("PYTHONPATH", "") + + +_CLAUDE_CASES = case_params( + _claude_driver_falls_back_to_transcript, + _claude_driver_writes_mcp_config_and_cmd_flags, + _claude_driver_server_command_override, + _claude_driver_timeout_returns_agent_run_not_raise, + _claude_driver_json_parse_failure_raises_for_infra_cli, + _claude_driver_uses_proxy_in_mcp_config, + _claude_driver_proxy_disabled_no_wrap, + _claude_mcp_env_has_pythonpath_when_proxied, +) + + +@pytest.mark.parametrize("case", _CLAUDE_CASES) +def test_claude_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def _known_drivers(): + assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + + +def _known_drivers_and_get_driver(): + assert "antigravity-cli" in KNOWN_DRIVERS + assert "opencode-cli" in KNOWN_DRIVERS + assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) + assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) + + +@pytest.mark.parametrize( + "case", + case_params(_known_drivers, _known_drivers_and_get_driver), +) +def test_known_drivers_behaviours(case): + case() def test_get_driver_api(): @@ -822,122 +833,127 @@ def test_get_driver_api(): assert isinstance(get_driver("codex-cli"), CodexCliDriver) -def test_antigravity_behaviours(tmp_path): - def test_antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path): - seen: dict = {} - - def fake_run(cmd, **kwargs): - seen["cmd"] = cmd - env = kwargs.get("env") or {} - seen["env"] = env - home = env.get("HOME") - if home: - cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" - seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None - return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "do it", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, - model="gemini-2.5", - max_turns=5, - cwd=tmp_path, - ) - assert seen["cmd"][0] == "agy" - assert "-p" in seen["cmd"] - assert "--output-format" in seen["cmd"] - assert "json" in seen["cmd"] - assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] - assert "no_turn_cap" in run.notes - assert seen.get("mcp_cfg") is not None - assert "mcpServers" in seen["mcp_cfg"] - assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) - - def test_antigravity_fallback_runner_timeout_harvests(tmp_path): - call_row = { - "tool": "g_tool", - "args": {}, - "is_error": False, - "result_chars": 1, - "duration_ms": 1, - "seq": 1, - } - meta = { - "row_type": "proxy_meta", - "relayed_lines": 1, - "unparsed_lines": 0, - "unmatched_responses": 0, - "notifications": 0, - "pending_left": 0, - "child_killed": False, - } +def _antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path): + seen: dict = {} - def fake_run(cmd, **kwargs): - run_env = kwargs.get("env") or {} - home = run_env.get("HOME") - if home: - # First attempt includes env= — plant sidecar from dual-written mcp config, - # then reject env so the driver retries without it. - for rel in ( - Path(home) / ".gemini" / "config" / "mcp_config.json", - Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", - ): - if rel.is_file(): - cfg = json.loads(rel.read_text()) - args = cfg["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - side.write_text( - "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", - encoding="utf-8", - ) - break - raise TypeError("runner does not accept env=") - # Fallback call (no env) times out — outer except must still harvest. - raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) - - driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) - run = driver.run_task( - "hi", - mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, - model=None, - max_turns=1, - cwd=tmp_path, - ) - assert run.stopped_reason == "timeout" - assert run.call_source == "proxy" - assert len(run.calls) == 1 - assert run.calls[0]["tool"] == "g_tool" - - _d0 = tmp_path / "test_antigravity_driver_writes_mcp_config_under_isolated_home" - _d0.mkdir() - test_antigravity_driver_writes_mcp_config_under_isolated_home(_d0) - _d1 = tmp_path / "test_antigravity_fallback_runner_timeout_harvests" - _d1.mkdir() - test_antigravity_fallback_runner_timeout_harvests(_d1) - - -def test_write_behaviours(tmp_path): - def test_write_antigravity_mcp_config_shape(tmp_path): - p = tmp_path / "mcp_config.json" - write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) - data = json.loads(p.read_text()) - assert data["mcpServers"]["plane"]["command"] == "python" - assert data["mcpServers"]["plane"]["env"]["A"] == "1" - - def test_write_opencode_mcp_config_shape(tmp_path): - p = tmp_path / "opencode.json" - write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) - data = json.loads(p.read_text()) - assert data["mcp"]["plane"]["command"][0] == "py" - assert data["mcp"]["plane"]["environment"]["K"] == "V" - - _d0 = tmp_path / "test_write_antigravity_mcp_config_shape" - _d0.mkdir() - test_write_antigravity_mcp_config_shape(_d0) - _d1 = tmp_path / "test_write_opencode_mcp_config_shape" - _d1.mkdir() - test_write_opencode_mcp_config_shape(_d1) + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + env = kwargs.get("env") or {} + seen["env"] = env + home = env.get("HOME") + if home: + cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" + seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, + model="gemini-2.5", + max_turns=5, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "agy" + assert "-p" in seen["cmd"] + assert "--output-format" in seen["cmd"] + assert "json" in seen["cmd"] + assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] + assert "no_turn_cap" in run.notes + assert seen.get("mcp_cfg") is not None + assert "mcpServers" in seen["mcp_cfg"] + assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) + + +def _antigravity_fallback_runner_timeout_harvests(tmp_path): + call_row = { + "tool": "g_tool", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + } + + def fake_run(cmd, **kwargs): + run_env = kwargs.get("env") or {} + home = run_env.get("HOME") + if home: + # First attempt includes env= — plant sidecar from dual-written mcp config, + # then reject env so the driver retries without it. + for rel in ( + Path(home) / ".gemini" / "config" / "mcp_config.json", + Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + cfg = json.loads(rel.read_text()) + args = cfg["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text( + "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", + encoding="utf-8", + ) + break + raise TypeError("runner does not accept env=") + # Fallback call (no env) times out — outer except must still harvest. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "g_tool" + + +@pytest.mark.parametrize( + "case", + case_params( + _antigravity_driver_writes_mcp_config_under_isolated_home, + _antigravity_fallback_runner_timeout_harvests, + ), +) +def test_antigravity_behaviours(case, tmp_path): + case(tmp_path) + + +def _write_antigravity_mcp_config_shape(tmp_path): + p = tmp_path / "mcp_config.json" + write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) + data = json.loads(p.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + assert data["mcpServers"]["plane"]["env"]["A"] == "1" + + +def _write_opencode_mcp_config_shape(tmp_path): + p = tmp_path / "opencode.json" + write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) + data = json.loads(p.read_text()) + assert data["mcp"]["plane"]["command"][0] == "py" + assert data["mcp"]["plane"]["environment"]["K"] == "V" + + +@pytest.mark.parametrize( + "case", + case_params(_write_antigravity_mcp_config_shape, _write_opencode_mcp_config_shape), +) +def test_write_behaviours(case, tmp_path): + case(tmp_path) def test_opencode_driver_writes_project_config(tmp_path: Path): diff --git a/tests/fixtures/evals_schema_v0_rows.jsonl b/tests/fixtures/evals_schema_v0_rows.jsonl index eee1306a..e05df63d 100644 --- a/tests/fixtures/evals_schema_v0_rows.jsonl +++ b/tests/fixtures/evals_schema_v0_rows.jsonl @@ -1,2 +1,2 @@ -{"run_id": "7a637f5a54664d3eb2fff9ac5a53fb43", "ts": "2026-08-12T17:59:05.498671+00:00", "git_sha": "5da71142cab2d9fd7e8f95be8192ccb17ac3d826", "battery": "6647676edc9e", "label": "manish-v2", "driver": "codex-cli", "server": "external", "model": "gpt-5.6-sol", "task_id": "L3", "author": "post-hoc-debias", "rep": 0, "success": true, "verify_note": "release tag 'eval-rc1' present", "skipped": null, "error": null, "error_class": null, "stop_reason": "end_turn", "hit_max_iterations": false, "calls": [{"tool": "release_tag", "class": "out_of_set", "args_chars": 43, "result_tokens": null, "result_chars": 1016, "result_kind": "text", "is_error": false, "duration_ms": 91, "action": "create", "result_tokens_skipped": "no API key / CLI driver has no count_tokens"}], "num_calls": 1, "errored_calls": 0, "alternate_calls": null, "out_of_set_calls": null, "total_result_tokens": 0, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 25.381, "client_tool_calls": [{"tool": "release_tag", "args_chars": 43, "raw_tool": "release_tag"}], "client_tool_call_count": 1, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_pair_mismatch": false, "token_count_failures": 0, "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff720-bf7b-75d2-9b23-eb0b635be673", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"], "result_tokens_skipped_reason": "CLI driver: count_tokens requires Anthropic API key; skipped", "usage": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 143874, "output_tokens": 504, "cache_read_input_tokens": 119552, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 263426, "source": "codex_token_count"}} -{"run_id": "625464c995c646429f7cfbcb1a9f5166", "ts": "2026-08-13T03:37:23.554029+00:00", "git_sha": "adf653458ed5788e58acc5e2e9751143df942a5d", "battery": "6425dcc64404", "label": "full", "driver": "codex-cli", "provider": null, "server": "local", "model": "gpt-5.6-sol", "requested_model": "gpt-5.6-sol", "task_id": "R2", "author": "claude", "rep": 0, "success": true, "verify_note": "final text names count 4", "skipped": null, "error": null, "error_class": null, "final_text": "I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4", "stop_reason": "end_turn", "hit_max_iterations": false, "result_pair_mismatch": false, "token_count_failures": 0, "result_tokens_estimated": true, "calls": [{"tool": "list_projects", "class": "alternate", "args_chars": 18, "result_tokens": 315, "result_chars": 1258, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 159}, {"tool": "count_work_items", "class": "optimal", "args_chars": 118, "result_tokens": 64, "result_chars": 253, "result_kind": "text", "is_error": false, "result_tokens_estimated": true, "result_token_count_method": "chars_div_4", "duration_ms": 107}], "num_calls": 2, "errored_calls": 0, "alternate_calls": 1, "out_of_set_calls": 0, "total_result_tokens": 379, "usage_per_iteration": [], "cum_input_tokens": null, "wall_time_s": 29.679, "client_tool_calls": [{"tool": "list_projects", "args_chars": 18, "raw_tool": "list_projects"}, {"tool": "count_work_items", "args_chars": 118, "raw_tool": "count_work_items"}], "client_tool_call_count": 2, "cum_input_tokens_reason": "CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting", "result_tokens_mode": "estimated", "result_token_count_method": "chars_div_4", "usage_scope": "run", "call_source": "proxy", "driver_raw_ref": "session:019ff932-5598-7d62-9ab9-30c6bf5fca15", "driver_notes": ["experimental:codex-cli", "proxy_sidecar_incomplete:no_meta", "calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"], "usage": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_tokens": null}, "usage_total": {"input_tokens": 157010, "output_tokens": 501, "cache_read_input_tokens": 135680, "cache_creation_input_tokens": 0, "total_input_tokens_including_cache": 292690, "source": "codex_token_count"}} +{"run_id":"7a637f5a54664d3eb2fff9ac5a53fb43","ts":"2026-08-12T17:59:05.498671+00:00","git_sha":"5da71142cab2d9fd7e8f95be8192ccb17ac3d826","battery":"6647676edc9e","label":"manish-v2","driver":"codex-cli","server":"external","model":"gpt-5.6-sol","task_id":"L3","author":"post-hoc-debias","rep":0,"success":true,"verify_note":"release tag 'eval-rc1' present","skipped":null,"error":null,"error_class":null,"stop_reason":"end_turn","hit_max_iterations":false,"calls":[{"tool":"release_tag","args_chars":43,"result_tokens":null,"result_chars":1016,"result_kind":"text","is_error":false,"duration_ms":91,"action":"create","result_tokens_skipped":"no API key / CLI driver has no count_tokens"}],"num_calls":1,"errored_calls":0,"total_result_tokens":0,"usage_per_iteration":[],"cum_input_tokens":null,"wall_time_s":25.381,"client_tool_calls":[{"tool":"release_tag","args_chars":43,"raw_tool":"release_tag"}],"client_tool_call_count":1,"cum_input_tokens_reason":"CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting","result_pair_mismatch":false,"token_count_failures":0,"usage_scope":"run","call_source":"proxy","driver_raw_ref":"session:019ff720-bf7b-75d2-9b23-eb0b635be673","driver_notes":["experimental:codex-cli","proxy_sidecar_incomplete:no_meta","calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"],"result_tokens_skipped_reason":"CLI driver: count_tokens requires Anthropic API key; skipped","usage":{"input_tokens":143874,"output_tokens":504,"cache_read_input_tokens":119552,"cache_creation_input_tokens":0,"total_tokens":null},"usage_total":{"input_tokens":143874,"output_tokens":504,"cache_read_input_tokens":119552,"cache_creation_input_tokens":0,"total_input_tokens_including_cache":263426,"source":"codex_token_count"}} +{"run_id":"625464c995c646429f7cfbcb1a9f5166","ts":"2026-08-13T03:37:23.554029+00:00","git_sha":"adf653458ed5788e58acc5e2e9751143df942a5d","battery":"6425dcc64404","label":"full","driver":"codex-cli","provider":null,"server":"local","model":"gpt-5.6-sol","requested_model":"gpt-5.6-sol","task_id":"R2","author":"claude","rep":0,"success":true,"verify_note":"final text names count 4","skipped":null,"error":null,"error_class":null,"final_text":"I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4","stop_reason":"end_turn","hit_max_iterations":false,"result_pair_mismatch":false,"token_count_failures":0,"result_tokens_estimated":true,"calls":[{"tool":"list_projects","args_chars":18,"result_tokens":315,"result_chars":1258,"result_kind":"text","is_error":false,"result_tokens_estimated":true,"result_token_count_method":"chars_div_4","duration_ms":159},{"tool":"count_work_items","args_chars":118,"result_tokens":64,"result_chars":253,"result_kind":"text","is_error":false,"result_tokens_estimated":true,"result_token_count_method":"chars_div_4","duration_ms":107}],"num_calls":2,"errored_calls":0,"total_result_tokens":379,"usage_per_iteration":[],"cum_input_tokens":null,"wall_time_s":29.679,"client_tool_calls":[{"tool":"list_projects","args_chars":18,"raw_tool":"list_projects"},{"tool":"count_work_items","args_chars":118,"raw_tool":"count_work_items"}],"client_tool_call_count":2,"cum_input_tokens_reason":"CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting","result_tokens_mode":"estimated","result_token_count_method":"chars_div_4","usage_scope":"run","call_source":"proxy","driver_raw_ref":"session:019ff932-5598-7d62-9ab9-30c6bf5fca15","driver_notes":["experimental:codex-cli","proxy_sidecar_incomplete:no_meta","calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"],"usage":{"input_tokens":157010,"output_tokens":501,"cache_read_input_tokens":135680,"cache_creation_input_tokens":0,"total_tokens":null},"usage_total":{"input_tokens":157010,"output_tokens":501,"cache_read_input_tokens":135680,"cache_creation_input_tokens":0,"total_input_tokens_including_cache":292690,"source":"codex_token_count"}} From cd900bc1063b54bc9bda660c411c1eb4e2e3e570 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 18:32:58 +0530 Subject: [PATCH 36/93] Restore per-case test reporting, and document what changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folding duplicated cases into nested closures halved the suite but cost diagnosability: a failing case aborted its siblings, so one regression hid the rest, and pytest reported one test instead of fourteen. Case tables move to parametrize — source stays consolidated, execution and reporting go back per-case. A single broken behaviour now surfaces eleven distinct failures. Docs record the fingerprint's meaning (task id, prompt and fixture names — what the agent was asked and what it was given, never how it should answer), the revision history and why each bump happened, and that exit 0 now means the evaluation completed cleanly rather than that the agent passed. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 95 ++-- evals/README.md | 90 ++-- evals/cli.py | 36 +- evals/runner/live.py | 9 +- tests/evals/conftest.py | 5 + tests/evals/test_cli.py | 117 ++--- tests/evals/test_docs.py | 23 +- tests/evals/test_import_compat.py | 33 ++ tests/evals/test_proxy.py | 792 +++++++++++++++-------------- tests/evals/test_results.py | 361 +++++++------ tests/evals/test_skip_taxonomy.py | 54 ++ tests/evals/test_token_counting.py | 103 ++-- 12 files changed, 944 insertions(+), 774 deletions(-) create mode 100644 tests/evals/test_import_compat.py create mode 100644 tests/evals/test_skip_taxonomy.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 32cb19bd..4f4f64c7 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -10,23 +10,24 @@ This document explains why the harness is shaped this way. Operational commands ## The questions it answers -The original tool-consolidation question breaks down into three measurable questions: - -1. **Mispick rate** — how often does an agent choose an alternate or out-of-set tool when - several tools have overlapping names or capabilities? -2. **Calls-to-done versus optimal** — how much lookup, name-to-ID resolution, and - sub-object fan-out does the surface require before the task is complete? -3. **Response bloat** — how much tool-result content is injected into the conversation? - -Success is the guardrail around all three. A surface that uses fewer calls or returns less -text but fails the task is not an improvement. Conversely, success rate alone hides -avoidable calls, wrong turns, and large responses. The harness therefore records all four +The original tool-consolidation question breaks down into four measurable questions: + +1. **Task success** — did the agent produce the verified Plane state or exact answer? +2. **Calls to done** — how much lookup, name-to-ID resolution, and sub-object fan-out did + successful repetitions actually require? +3. **Tool-use stability** — which tools were core across successful repetitions, and where + did repetitions choose different routes? +4. **Response bloat** — how much tool-result content was injected into the conversation? + +Success is the guardrail around the other three. A surface that uses fewer calls or returns +less text but fails the task is not an improvement. Conversely, success rate alone hides +extra calls, variable routes, and large responses. The harness therefore records all four dimensions for the same task execution. The point is empirical comparison. Given the same task battery, model, and repetitions, different surfaces can be compared from observed behavior rather than from tool counts, -schema inspection, or projected costs. The battery fingerprint records the prompt and -tool-set definition used for a run so incompatible batteries are not silently compared. +schema inspection, or projected costs. The battery fingerprint records task IDs and prompts +so incompatible batteries are not silently compared. ## What is measured @@ -39,38 +40,60 @@ exact-value matchers where the task defines them. This avoids using the agent's explanation, confidence, or self-reported completion as the source of truth. The model is also not asked to grade another model. Verification is tied to -the fixture and the Plane state the task was meant to affect. The canary runs every eligible -verifier against an empty agent result and fails if a do-nothing run passes. +the fixture and the Plane state the task was meant to affect. The canary reports which +verifiers were exercised, skipped, or errored and probes eligible verifiers with an empty +result plus plausible zero-call contract answers. CI can name an explicit strict set of task +ids that must be eligible in its environment. Skipped tasks and infrastructure failures are recorded separately. The report excludes both from success denominators; a plan gate, unavailable fixture, provider failure, or MCP process failure is not rewritten as an agent task failure. +Caught exceptions follow one validity convention: continuing is allowed only when a local +fallback makes the result equivalent, and that catch documents why. Failures that can alter +the evaluated state, recorded evidence, cleanup, or report denominator are represented as +infrastructure, harness, or cleanup errors so run completeness cannot silently remain green. + ### Calls to done -`num_calls` counts Plane MCP calls made during the task. Each catalog entry also declares an -`optimal_calls` baseline. The report shows the observed distribution rather than assuming -one run is representative. +`num_calls` counts Plane MCP calls made during the task. The report shows the observed +distribution rather than assuming one run is representative. Every reported call-count +minimum, median, maximum, and Q1–Q3 span is conditioned on successful repetitions; a run +that failed early is not treated as a cost-to-success observation. There is no +author-declared call floor. + +Two-label reports pair tasks before making inferential comparisons. Their success-rate +difference uses a paired percentile-bootstrap interval that resamples tasks as the +independent units. Their mean call-count delta uses a paired sign-flip permutation test on +the actual magnitudes, retaining zero-delta ties. These procedures assume comparable task +instances under the two labels, independent tasks, and exchangeable A/B labels under the +permutation null; they do not account for shared environment drift or dependence between +tasks. The report prints the paired task count so small samples remain visible. Client-local tools such as shell or tool-search helpers are retained separately as `client_tool_calls`; they do not count as Plane calls. For an external server launched with -`--server-cmd`, call counts still apply, but the runner marks the row server as `external` -and clears the row-level alternate/out-of-set counters because the catalog has no -authoritative sets for foreign tool names. +`--server-cmd`, the runner marks the row server as `external`; call counts and observed tool +distributions use the same rules as local-server rows. -### Mispicks +### Observed tool distribution -Every Plane call on a catalogued surface is classified by tool name as `optimal`, -`alternate`, or `out_of_set`. The task owns disjoint optimal and alternate sets. The -headline mispick rate is: +The former author-declared optimal/alternate sets and mispick score were removed. Reports +now describe the tools agents used in successful repetitions: -```text -(alternate calls + out-of-set calls) / all Plane calls -``` +- `tool_rep_frequency` is the share of successful repetitions that used each tool at least + once. Repeated calls in one repetition count once for frequency. +- `tool_call_counts` is the total number of calls to each tool across those repetitions. +- Reports label the successful-repetition denominator as `success-only n=...` and show the + number of non-success, non-skip repetitions omitted from it as `failed excluded=...`. + The exclusion count includes recorded harness/infrastructure errors; skips did not run + the agent and remain in execution coverage instead. -`is_error` is independent of that classification. A valid call can still be an avoidable -pick, and an optimal tool can return an error. The ordered call records are retained in the -JSONL so a run can be audited after aggregation. +Failed repetitions are excluded because an early failure would otherwise make the tools in +successful runs appear variable. With fewer than two successful repetitions, variance is +not observable and the report shows `frequency=—` beside those counts. Frequency `1.0` is rendered as core use; lower +positive frequency is variable use. The fleet headline counts tasks with at least one +variable tool. Because the measurement is descriptive, external servers get the same metric +as local servers even when their tool names differ. ### Response-token cost @@ -93,6 +116,14 @@ Payload recording is off by default because tool results contain live workspace make sidecars larger. The character-derived estimate remains useful for surface comparison because it is deterministic and monotonic in the recorded response size. +Read-task provenance does not turn payload recording back on. Each read seeder registers a +hidden, per-run target-entity sentinel. At the API loop or CLI recording proxy, the harness +matches that sentinel while the successful response is in memory and persists only a +non-sensitive `observed_sentinels` label. A successful unrelated Plane call therefore does +not count as evidence, and neither the response body nor the sentinel value enters the +payload-free result row. A driver path without response matching is reported as unavailable +and cannot pass a read verifier. + Provider usage is a different measurement: where the driver supplies it, the harness keeps input, output, cache-read, and cache-creation usage. Tool-result sizing describes one source of context growth; it is not substituted for the provider's conversation-level usage. @@ -100,7 +131,7 @@ of context growth; it is not substituted for the provider's conversation-level u ## Why calls are recorded at the transport boundary An agent's final answer is not a reliable call log. It may omit a failed lookup, summarize -several calls as one action, or claim an action it did not perform. Call-count and mispick +several calls as one action, or claim an action it did not perform. Call-count and tool-use metrics therefore come from execution evidence. The API driver owns the MCP session and records each call it executes. The four CLI drivers diff --git a/evals/README.md b/evals/README.md index 481f1b08..56d97bc4 100644 --- a/evals/README.md +++ b/evals/README.md @@ -10,9 +10,9 @@ The harness is agent-agnostic and surface-agnostic: any stdio MCP server can be (`--server-cmd`), driven by any of five driver implementations. `DESIGN.md` explains why it is built this way; this file is how to run it. -What you get per task: pass/fail, tool calls to done, which tools were picked (and whether -they were the optimal ones), response size and token-count provenance, errors, and the -agent's final text. +What you get per task: pass/fail, observed calls to done, core and variable tool use across +successful repetitions, response size and token-count provenance, errors, and the agent's +final text. ## Prerequisites @@ -52,13 +52,12 @@ agent's final text. --server-env KEY=VALUE --out evals/output/their-pr.jsonl ``` -Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed or -skipped `(task, rep, label)` keys and retry rows with recorded errors), `--list` / `--dry-run` -(no network). +Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed +`(task, rep, label)` keys and plan-gated skips; retry recorded errors, cleanup failures, +fixture-collision skips, and unknown skips), `--list` / `--dry-run` (no network). -**External mode** (`--server-cmd`) records `server: "external"`. Foreign tool names have no catalogued optimal/alternate sets, -so use success, call counts, and errors for those rows; their mispick values are not -comparable to catalogued surfaces. +**External mode** (`--server-cmd`) records `server: "external"`. Success, call counts, +errors, and observed tool distributions use the same rules as local-server rows. ### Drivers @@ -113,6 +112,12 @@ workspace data and bloat the sidecar. For comparing tool surfaces, the default c estimate is monotonic in the thing being compared anyway. Do not enable payload recording by habit; use it only when the more sensitive, larger sidecar is justified. +Read-task provenance is stricter than “a call happened.” Seeders place a hidden per-run +sentinel on the target entity, and the API driver or CLI proxy records only whether a +successful response exposed it. Result rows contain the matched sentinel label, never the +sentinel value or response body. Thus an unrelated successful call cannot satisfy provenance; +a driver path where response matching is unavailable is diagnosed and fails closed. + ### Reading results ```bash @@ -127,26 +132,33 @@ denominators, as are rows with recorded errors. Result-token columns use `~` for `*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not recorded. +Reports keep three verdicts separate: model success among evaluated rows, execution coverage +(evaluated rows / expected rows, including skipped task IDs and capability reasons), and run +completeness. A plan-gated-only run can therefore be **RUN COMPLETE** below 100% execution +coverage. The live runner and report commands exit 0 when the evaluation completed cleanly; +exit 0 does **not** mean the agent passed. Callers that need a pass-threshold exit must apply +that as a separate opt-in policy rather than overloading the completeness status. + With `--reps N`, each `(task, rep)` is independently seeded, run, verified, and torn down. Multi-rep reports show each task's pass count, Wilson interval, and whether its pass/fail -answer changed across completed repetitions. The measured noise-floor line converts those -flips into task-count units: if `U` tasks were unstable, surface differences of `U` tasks or -fewer should be treated as within observed run-to-run variance, making `U + 1` the minimum -meaningful difference from that sample. This is an empirical guardrail, not proof that larger -differences are statistically significant. - -Every result row carries a `battery` fingerprint derived from the selected catalog's prompts -and tool metadata. Compare rows only when their fingerprints match: a table that mixes -fingerprints is comparing different questions, even when task IDs are the same. In particular, -results from a task whose output contract changed are not directly comparable with its rows in -older batteries. `evals.report --table` warns when its input rows span fingerprints. - -The hash covers prompts and tool sets, not fixtures or verifier bodies — prompt drift is what -it was built to catch. That leaves a hole, because correcting a seeder changes the question a -task puts to the agent without touching either. `CATALOG_REVISION` in `tasks/catalog.py` closes -it: bump it whenever a fixture or verifier change redefines what a task asks, and the -fingerprint moves with it. Revision 1 covers batteries 6-8; revision 2 is the feature-exclusion -correction, which made S5 genuinely require all three of its conditions. +answer changed across completed repetitions. Instability remains descriptive; it is not +converted into an ad-hoc threshold for declaring surface differences meaningful. Two-file +A/B reports instead pair shared tasks, report a paired-bootstrap 95% interval for the mean +per-task success-rate difference, and use a paired sign-flip permutation test for mean call +deltas. Zero call-delta ties remain in that paired sample. The inference treats tasks as +independent sampling units and assumes comparable task instances under both labels, so the +printed paired task count—and the resulting wide interval for small samples—matters. + +Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, +prompts, and catalog revision. It contains exactly what the agent is asked and no expectation +about how the answer should be produced. A table that mixes fingerprints normally compares +different questions, even when task IDs are the same, so `evals.report --table` warns when its +input rows span fingerprints. A revision can document a deliberately comparable structural +change when prompts and verifiers remain unchanged. + +The hash excludes fixtures and verifier bodies. `CATALOG_REVISION` in `tasks/catalog.py` +closes that gap: bump it whenever an excluded change redefines what a task asks, and explain +the comparison consequence in its docstring. ## Running surfaces in parallel @@ -188,9 +200,6 @@ A task is a dict: "id": "W11", "tags": {"write", "tier1"}, "prompt": f"In project {{project}}, ...", # {project} is bound at run time - "optimal_calls": 3, - "optimal_tools": {"list_cycles", "complete_cycle"}, # scored as optimal picks - "alternate_tools": {"list_projects"}, # acceptable, not optimal "needs": {"items", "cycles"}, # fixtures to seed "verify": verify_w11, } @@ -223,11 +232,15 @@ Then prove the verifier can fail: ```bash .venv/bin/python -m evals --canary --label local +# CI capability contract: these ids must be eligible and verified. +.venv/bin/python -m evals --canary --canary-strict R1,R2,W8 --label local ``` -The canary seeds each task, calls its verifier with an **empty** agent -result, and exits non-zero if any verifier passes a do-nothing agent. Run it after touching -tasks, fixtures, or verifiers. +The canary seeds each task, calls its verifier with both an **empty** agent result and +plausible zero-call canned contract answers, then reports verified, skipped, and errored +task ids separately. It exits non-zero for a false pass, verifier/teardown error, zero +verified tasks, or a skipped id named by `--canary-strict`. Plan-gated skips outside that +explicit strict set remain allowed. Run it after touching tasks, fixtures, or verifiers. **Make the task achievable before blaming a surface.** For example, W6 declares the `cycles_open_past` fixture variant because it asks the agent to close Sprint 12; the seeder @@ -254,10 +267,19 @@ Keep such scripts outside version control — `localdev/` is ignored for exactly - If seeded comments do not materialize as activities, the activity-feed task self-skips with `env:no-activity-worker` rather than failing the agent. -- A capability the workspace's plan excludes self-skips with `env:plan-gated:`. +- A reviewed capability the workspace's plan excludes self-skips with + `env:plan-gated:`. The closed allowlist is `customers`, `releases`, + `work-item-types`, `initiatives`, and `teamspaces`; a typo or new name is unexpected + until its real gate site is reviewed and the allowlist is deliberately extended. Only a refusal that names a plan limit counts: 402, or 403/400 whose body says so. A bare 403 is an ordinary permission denial and stays a real error, because classifying it as a gate would let a permission bug leave the denominator and read as "nothing to see". +- Run completeness uses an explicit skip taxonomy: known capabilities the environment does + not provide (an allowlisted `env:plan-gated:` or the exact reason + `env:no-activity-worker`) are expected skips. + They reduce **EXECUTION COVERAGE** but do not break **RUN COMPLETE**. A dirty environment + (`env:fixture-collision:*`) and every unrecognised reason are unexpected and make the run + incomplete; there is intentionally no catch-all for new `env:*` reasons. - A feature switched **off for a project** is not a plan gate — it is configuration the harness sets itself, and W11 exists to measure what an agent does when it meets one. - **Gated endpoints returning 402 on a workspace that should work.** Feature flags are diff --git a/evals/cli.py b/evals/cli.py index 134efd5d..3c17f3be 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -113,7 +113,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=None, help=( "External MCP stdio server launch command (shlex-split). Enables external " - "mode, where foreign tool names make mispick classification unavailable." + "mode while retaining the same observed tool-use metrics." ), ) p.add_argument( @@ -155,8 +155,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: metavar="OUT.jsonl", help=( "Resume into an existing JSONL (also the --out target). Skip " - "(task_id, rep, label) keys that already completed; re-run rows with infra_ " - "error_class or non-null error." + "(task_id, rep, label) keys that completed or were plan-gated; re-run rows " + "with errors, cleanup failures, fixture collisions, or unknown skips." ), ) p.add_argument( @@ -167,6 +167,16 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "(no driver/model), teardown. Exit 1 if any verifier returns ok=True on do-nothing." ), ) + p.add_argument( + "--canary-strict", + type=str, + default=None, + metavar="TASK_IDS", + help=( + "Strict canary coverage: comma-separated task ids that must be verified. " + "Plan-gated skips outside this explicit eligible set remain allowed." + ), + ) return p.parse_args(argv) @@ -177,14 +187,14 @@ def _task_ids(raw: str | None) -> list[str] | None: def cmd_list() -> int: - print(f"{'id':<6} {'tags':<18} {'opt':>4} prompt") + print(f"{'id':<6} {'tags':<18} prompt") print("-" * 100) for task in TASKS: tags = ",".join(sorted(task["tags"])) prompt = task["prompt"].replace("\n", " ") if len(prompt) > 70: prompt = prompt[:67] + "..." - print(f"{task['id']:<6} {tags:<18} {task['optimal_calls']:>4} {prompt}") + print(f"{task['id']:<6} {tags:<18} {prompt}") return 0 @@ -202,8 +212,6 @@ def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: print(f"=== {task['id']} ===") print(f"needs: {sorted(task.get('needs') or [])}") print(f"author: {task.get('author') or 'claude'}") - print(f"optimal_calls: {task['optimal_calls']}") - print(f"optimal_tools: {sorted(task['optimal_tools'])}") print(f"prompt:\n {resolved}") print() return 0 @@ -247,7 +255,19 @@ def main(argv: list[str] | None = None) -> int: # Canary: live env only — no driver/model required. if args.canary: - return asyncio.run(run_canary(tasks, label=label)) + required_ids = set(_task_ids(args.canary_strict) or []) + if args.canary_strict is not None and not required_ids: + print("error: --canary-strict requires at least one task id", file=sys.stderr) + return 2 + known_ids = {str(task["id"]) for task in TASKS} + unknown_required = sorted(required_ids - known_ids) + if unknown_required: + print(f"error: unknown --canary-strict task id(s): {', '.join(unknown_required)}", file=sys.stderr) + return 2 + return asyncio.run(run_canary(tasks, label=label, required_task_ids=required_ids)) + if args.canary_strict is not None: + print("error: --canary-strict requires --canary", file=sys.stderr) + return 2 driver_name = (getattr(args, "driver", None) or "api").strip().lower() if driver_name not in KNOWN_DRIVERS: diff --git a/evals/runner/live.py b/evals/runner/live.py index 8c5e6fb5..0a4ddc76 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -14,7 +14,7 @@ from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS -from evals.evidence import normalize_evidence_sentinels +from evals.evidence import configured_evidence_labels from evals.report.load import load_rows from evals.report.summary import completeness_statement, execution_coverage_statement, summarize from evals.results import TaskResult, agent_run_to_task_result @@ -116,6 +116,7 @@ async def run_agent_task_via_driver( system=system, cwd=Path(__file__).resolve().parent.parent.parent, evidence_sentinels=ctx.get("evidence_sentinels"), + evidence_targets=ctx.get("evidence_targets"), ) return agent_run_to_task_result(agent_run) @@ -175,10 +176,10 @@ def _seed_fixtures( ctx=context, task_id=str(task["id"]), ) - if "read" in set(task.get("tags") or set()) and not normalize_evidence_sentinels( - context.get("evidence_sentinels") + if "read" in set(task.get("tags") or set()) and not configured_evidence_labels( + context.get("evidence_sentinels"), context.get("evidence_targets") ): - raise RuntimeError(f"{task['id']} seed did not register target-entity evidence sentinels") + raise RuntimeError(f"{task['id']} seed did not register target-bound response evidence") except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason diff --git a/tests/evals/conftest.py b/tests/evals/conftest.py index 253408dc..9cf0f3f4 100644 --- a/tests/evals/conftest.py +++ b/tests/evals/conftest.py @@ -8,6 +8,11 @@ import pytest +def case_params(*cases): + """Build readable pytest cases from consolidated case helpers.""" + return [pytest.param(case, id=case.__name__.removeprefix("_").replace("_", "-")) for case in cases] + + @pytest.fixture(autouse=True) def _eval_creds(monkeypatch): monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") diff --git a/tests/evals/test_cli.py b/tests/evals/test_cli.py index 36c1ca3e..ee77b269 100644 --- a/tests/evals/test_cli.py +++ b/tests/evals/test_cli.py @@ -35,53 +35,59 @@ EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features -def test_cmd_behaviours(capsys): - def test_cmd_list_prints_all_task_ids(capsys): - rc = cmd_list() - assert rc == 0 +@pytest.mark.parametrize("case", ["list-task-ids", "dry-run-all"]) +def test_cmd_behaviours(case, capsys): + if case == "list-task-ids": + assert cmd_list() == 0 out = capsys.readouterr().out - for tid in DESIGN_IDS | EXTRA_IDS: - assert tid in out - - def test_cmd_dry_run_all_tasks(capsys): - rc = cmd_dry_run(list(TASKS)) - assert rc == 0 + for task_id in DESIGN_IDS | EXTRA_IDS: + assert task_id in out + else: + assert cmd_dry_run(list(TASKS)) == 0 out = capsys.readouterr().out assert "Seed plan:" in out - for tid in ("R1", "W9", "S4", "C2", "R7"): - assert f"=== {tid} ===" in out - - test_cmd_list_prints_all_task_ids(capsys) - test_cmd_dry_run_all_tasks(capsys) + for task_id in ("R1", "W9", "S4", "C2", "R7"): + assert f"=== {task_id} ===" in out -def test_parse_args_behaviours(): - def test_parse_args_list(): - a = parse_args(["--list", "--label", "candidate-build"]) - assert a.list is True - assert a.label == "candidate-build" +@pytest.mark.parametrize("case", ["list", "driver", "resume-and-canary"]) +def test_parse_args_behaviours(case): + if case == "list": + args = parse_args(["--list", "--label", "candidate-build"]) + assert args.list is True + assert args.label == "candidate-build" assert parse_args(["--list"]).label == "local" + elif case == "driver": + assert parse_args(["--driver", "claude-cli", "--dry-run"]).driver == "claude-cli" + defaults = parse_args(["--dry-run"]) + assert (defaults.driver, defaults.model, defaults.provider) == ("api", "standard", "anthropic") + assert defaults.record_result_payloads is False + recorded = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) + assert recorded.record_result_payloads is True + else: + assert run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]).resume == "evals/output/x.jsonl" + assert run_mod.parse_args(["--canary", "--tasks", "R1"]).canary is True + assert run_mod.parse_args(["--canary", "--canary-strict", "R1,R2"]).canary_strict == "R1,R2" + + +def test_canary_strict_cli_passes_explicit_required_ids(monkeypatch): + seen: dict = {} + + async def fake_canary(tasks, *, label, required_task_ids): + seen.update(task_ids=[task["id"] for task in tasks], label=label, required=required_task_ids) + return 0 - def test_parse_args_accepts_driver(): - a = parse_args(["--driver", "claude-cli", "--dry-run"]) - assert a.driver == "claude-cli" - b = parse_args(["--dry-run"]) - assert b.driver == "api" - assert b.model == "standard" - assert b.provider == "anthropic" - assert b.record_result_payloads is False - c = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) - assert c.record_result_payloads is True + monkeypatch.setattr(run_mod, "run_canary", fake_canary) + rc = eval_main(["--canary", "--tasks", "R1,R2", "--canary-strict", "R1", "--label", "ci"]) + assert rc == 0 + assert seen == {"task_ids": ["R1", "R2"], "label": "ci", "required": {"R1"}} - def test_parse_args_resume_and_canary(): - a = run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]) - assert a.resume == "evals/output/x.jsonl" - b = run_mod.parse_args(["--canary", "--tasks", "R1"]) - assert b.canary is True - test_parse_args_list() - test_parse_args_accepts_driver() - test_parse_args_resume_and_canary() +def test_canary_strict_cli_rejects_invalid_usage(capsys): + assert eval_main(["--canary-strict", "R1"]) == 2 + assert "requires --canary" in capsys.readouterr().err + assert eval_main(["--canary", "--canary-strict", "NOPE"]) == 2 + assert "unknown --canary-strict" in capsys.readouterr().err def test_model_tiers_resolve_per_driver_and_provider(): @@ -112,35 +118,18 @@ def test_non_tier_model_strings_pass_through_unchanged(driver, model): assert resolve_model_for_driver(driver, model) == model -def test_unmapped_behaviours(tmp_path, capsys): - def test_unmapped_opencode_tier_fails_with_explicit_model_guidance(): +@pytest.mark.parametrize("case", ["direct-guidance", "cli-error"]) +def test_unmapped_behaviours(case, tmp_path, capsys): + if case == "direct-guidance": with pytest.raises(ValueError, match=r"opencode models"): resolve_model_for_driver("opencode-cli", "standard") + return - def test_unmapped_tier_cli_error_is_loud_and_prevents_run(tmp_path, capsys): - out = tmp_path / "must-not-exist.jsonl" - - rc = eval_main( - [ - "--driver", - "opencode-cli", - "--model", - "standard", - "--tasks", - "R1", - "--out", - str(out), - ] - ) - - assert rc == 2 - assert "explicit provider/model ID" in capsys.readouterr().err - assert out.exists() is False - - test_unmapped_opencode_tier_fails_with_explicit_model_guidance() - _d1 = tmp_path / "test_unmapped_tier_cli_error_is_loud_and_prevents_run" - _d1.mkdir() - test_unmapped_tier_cli_error_is_loud_and_prevents_run(_d1, capsys) + out = tmp_path / "must-not-exist.jsonl" + rc = eval_main(["--driver", "opencode-cli", "--model", "standard", "--tasks", "R1", "--out", str(out)]) + assert rc == 2 + assert "explicit provider/model ID" in capsys.readouterr().err + assert out.exists() is False def test_tier_mapping_is_scoped_to_cli_provider(): diff --git a/tests/evals/test_docs.py b/tests/evals/test_docs.py index 4a4d945e..f2e6af4f 100644 --- a/tests/evals/test_docs.py +++ b/tests/evals/test_docs.py @@ -7,14 +7,20 @@ from __future__ import annotations +import pytest + from evals import REPO_ROOT README = (REPO_ROOT / "evals" / "README.md").read_text() DESIGN = (REPO_ROOT / "evals" / "DESIGN.md").read_text() -def test_the_behaviours(): - def test_the_flag_server_is_documented_as_optional(): +@pytest.mark.parametrize( + "case", + ["optional-flag-server", "plan-gate-reason", "catalog-revision", "exit-zero-contract"], +) +def test_the_behaviours(case): + if case == "optional-flag-server": assert "FEATURE_FLAG_SERVER_BASE_URL" in README, "the option should still be documented" index = README.index("FEATURE_FLAG_SERVER_BASE_URL") paragraph = README[max(0, index - 200) : index + 500] @@ -22,16 +28,13 @@ def test_the_flag_server_is_documented_as_optional(): "the flag server stopped being a prerequisite when the seeders learned to skip; " "the runbook must not tell people otherwise" ) - - def test_the_plan_gate_skip_reason_is_documented(): + elif case == "plan-gate-reason": assert "env:plan-gated:" in README - - def test_the_fingerprint_revision_is_documented(): + elif case == "catalog-revision": assert "CATALOG_REVISION" in README - - test_the_flag_server_is_documented_as_optional() - test_the_plan_gate_skip_reason_is_documented() - test_the_fingerprint_revision_is_documented() + else: + assert "exit 0 does **not** mean the agent passed" in README + assert "execution coverage" in README def test_design_still_states_the_skip_contract_the_seeders_now_implement(): diff --git a/tests/evals/test_import_compat.py b/tests/evals/test_import_compat.py new file mode 100644 index 00000000..253b1059 --- /dev/null +++ b/tests/evals/test_import_compat.py @@ -0,0 +1,33 @@ +"""Public import compatibility after neutral fixture extraction.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def test_seed_and_task_packages_import_in_either_order_with_legacy_reexports(): + root = Path(__file__).parents[2] + assertions = """ +from evals.errors import TaskSkipped as NeutralTaskSkipped +from evals.fixtures import CUSTOMER_NAME as NeutralCustomerName +from evals.seed import CUSTOMER_NAME, R1_TITLE +from evals.seed.customers import is_evaluation_customer_name +from evals.seed.releases import EVALUATION_RELEASE_TAG_VERSION +from evals.tasks.skip import TaskSkipped +assert CUSTOMER_NAME == NeutralCustomerName == 'Acme Corp' +assert R1_TITLE == 'Payment webhook drops retries' +assert EVALUATION_RELEASE_TAG_VERSION == 'eval-rc1' +assert is_evaluation_customer_name('Acme') +assert TaskSkipped is NeutralTaskSkipped +""" + for imports in ("import evals.tasks\nimport evals.seed\n", "import evals.seed\nimport evals.tasks\n"): + result = subprocess.run( + [sys.executable, "-c", imports + assertions], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index b7aa95db..9c1f3da0 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -16,6 +16,7 @@ load_proxy_sidecar, load_proxy_sidecar_calls, ) +from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -26,6 +27,7 @@ write_all_fd, ) from evals.proxy import main as proxy_main +from tests.evals.conftest import case_params REPO = Path(__file__).resolve().parents[2] @@ -100,77 +102,77 @@ def _write_fake_server(path: Path) -> Path: return path -def test_proxy_behaviours(tmp_path): - def test_proxy_records_tools_call_and_exit_code(tmp_path): - server = _write_fake_server(tmp_path / "fake_server.py") - sidecar = tmp_path / "side.jsonl" - cmd = [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ] - # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. - client_in = ( - json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) - + "\n" - + json.dumps( - { - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": {"name": "list_work_items", "arguments": {"project": "P"}}, - } - ) - + "\n" - + json.dumps( - { - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": {"name": "boom", "arguments": {}}, - } - ) - + "\n" - + "NOT_JSON_LINE\n" +def _proxy_records_tools_call_and_exit_code(tmp_path): + server = _write_fake_server(tmp_path / "fake_server.py") + sidecar = tmp_path / "side.jsonl" + cmd = [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ] + # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. + client_in = ( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {"project": "P"}}, + } ) - proc = subprocess.run( - cmd, - input=client_in.encode("utf-8"), - capture_output=True, - cwd=str(REPO), - timeout=15, + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "boom", "arguments": {}}, + } ) - assert proc.returncode == 7 # child exit propagated - # Byte-faithful: unparsed line and JSON responses appear on stdout. - out = proc.stdout.decode("utf-8", errors="replace") - assert "NOT_JSON_LINE" in out - assert "list_work_items" in out or "ok:list_work_items" in out - - rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] - call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] - meta = next(r for r in rows if r.get("row_type") == "proxy_meta") - assert len(call_rows) == 2 - assert call_rows[0]["tool"] == "list_work_items" - assert call_rows[0]["args"] == {"project": "P"} - assert call_rows[0]["is_error"] is False - assert call_rows[0]["result_chars"] > 0 - assert call_rows[0]["seq"] == 1 - assert call_rows[1]["tool"] == "boom" - assert call_rows[1]["is_error"] is True - assert meta["unparsed_lines"] >= 1 - assert meta["relayed_lines"] >= 3 - - def test_proxy_byte_faithful_child_receives_exact_bytes(tmp_path): - received = tmp_path / "received.bin" - echo_server = tmp_path / "echo_server.py" - echo_server.write_text( - textwrap.dedent( - f""" + + "\n" + + "NOT_JSON_LINE\n" + ) + proc = subprocess.run( + cmd, + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7 # child exit propagated + # Byte-faithful: unparsed line and JSON responses appear on stdout. + out = proc.stdout.decode("utf-8", errors="replace") + assert "NOT_JSON_LINE" in out + assert "list_work_items" in out or "ok:list_work_items" in out + + rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert len(call_rows) == 2 + assert call_rows[0]["tool"] == "list_work_items" + assert call_rows[0]["args"] == {"project": "P"} + assert call_rows[0]["is_error"] is False + assert call_rows[0]["result_chars"] > 0 + assert call_rows[0]["seq"] == 1 + assert call_rows[1]["tool"] == "boom" + assert call_rows[1]["is_error"] is True + assert meta["unparsed_lines"] >= 1 + assert meta["relayed_lines"] >= 3 + + +def _proxy_byte_faithful_child_receives_exact_bytes(tmp_path): + received = tmp_path / "received.bin" + echo_server = tmp_path / "echo_server.py" + echo_server.write_text( + textwrap.dedent( + f""" import sys data = sys.stdin.buffer.read() open({str(received)!r}, "wb").write(data) @@ -191,105 +193,108 @@ def test_proxy_byte_faithful_child_receives_exact_bytes(tmp_path): ) sys.stdout.buffer.flush() """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "s.jsonl" - # Deliberately non-canonical JSON spacing — re-serialization would change it. - payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(echo_server), - ], - input=payload, - capture_output=True, - cwd=str(REPO), - timeout=10, - ) - assert proc.returncode == 0 - assert received.read_bytes() == payload + ), + encoding="utf-8", + ) + sidecar = tmp_path / "s.jsonl" + # Deliberately non-canonical JSON spacing — re-serialization would change it. + payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(echo_server), + ], + input=payload, + capture_output=True, + cwd=str(REPO), + timeout=10, + ) + assert proc.returncode == 0 + assert received.read_bytes() == payload - def test_proxy_main_requires_command(): - with pytest.raises(SystemExit): - proxy_main(["--log", "/tmp/x.jsonl"]) - def test_proxy_exits_when_child_dies_first(tmp_path): - server = tmp_path / "die_soon.py" - server.write_text( - textwrap.dedent( - """ +def _proxy_main_requires_command(_tmp_path): + with pytest.raises(SystemExit): + proxy_main(["--log", "/tmp/x.jsonl"]) + + +def _proxy_exits_when_child_dies_first(tmp_path): + server = tmp_path / "die_soon.py" + server.write_text( + textwrap.dedent( + """ import sys, time # Emit nothing and exit quickly; leave proxy client stdin open. time.sleep(0.15) sys.exit(3) """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - t0 = __import__("time").monotonic() - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=str(REPO), - ) + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + t0 = __import__("time").monotonic() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + # Keep stdin open (do not close) so the stdin pump blocks on readline; + # the proxy must still notice child death and exit. + deadline = SHUTDOWN_DEADLINE_S + 5.0 try: - # Keep stdin open (do not close) so the stdin pump blocks on readline; - # the proxy must still notice child death and exit. - deadline = SHUTDOWN_DEADLINE_S + 5.0 + rc = proc.wait(timeout=deadline) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + pytest.fail(f"proxy hung >{deadline}s after child exit") + elapsed = __import__("time").monotonic() - t0 + # Must finish well under the hang window (not wait the full drain). + assert elapsed < deadline + # Child's exit code (3) should propagate; tolerate signal map if the + # runtime reaps oddly, but meta must still be present. + assert rc in (3, 128 + 3) or rc == 3 + assert sidecar.is_file() + text = sidecar.read_text(encoding="utf-8") + assert "proxy_meta" in text + # Prefer exact child code when available + if rc not in (3, 128 + 3): + # At least ensure we did not hang; surface stderr for diagnosis. + err = (proc.stderr.read() if proc.stderr else b"").decode() + assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.stdin: try: - rc = proc.wait(timeout=deadline) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - pytest.fail(f"proxy hung >{deadline}s after child exit") - elapsed = __import__("time").monotonic() - t0 - # Must finish well under the hang window (not wait the full drain). - assert elapsed < deadline - # Child's exit code (3) should propagate; tolerate signal map if the - # runtime reaps oddly, but meta must still be present. - assert rc in (3, 128 + 3) or rc == 3 - assert sidecar.is_file() - text = sidecar.read_text(encoding="utf-8") - assert "proxy_meta" in text - # Prefer exact child code when available - if rc not in (3, 128 + 3): - # At least ensure we did not hang; surface stderr for diagnosis. - err = (proc.stderr.read() if proc.stderr else b"").decode() - assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" - finally: - if proc.poll() is None: - proc.kill() - proc.wait() - if proc.stdin: - try: - proc.stdin.close() - except Exception: - pass + proc.stdin.close() + except Exception: + pass - def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path): - server = tmp_path / "echo_once.py" - server.write_text( - textwrap.dedent( - """ + +def _proxy_from_foreign_cwd_with_pythonpath(tmp_path): + server = tmp_path / "echo_once.py" + server.write_text( + textwrap.dedent( + """ import json, sys line = sys.stdin.readline() msg = json.loads(line) @@ -300,53 +305,54 @@ def test_proxy_from_foreign_cwd_with_pythonpath(tmp_path): }) + "\\n") sys.stdout.flush() """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - foreign = tmp_path / "foreign_cwd" - foreign.mkdir() - env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) - # Drop any ambient PYTHONPATH pollution by putting repo first. - assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) - client_in = ( - json.dumps( - { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "ping", "arguments": {}}, - } - ) - + "\n" - ) - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - input=client_in.encode(), - capture_output=True, - cwd=str(foreign), # foreign cwd — must still import evals.proxy - env=env, - timeout=15, + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign_cwd" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) + # Drop any ambient PYTHONPATH pollution by putting repo first. + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "ping", "arguments": {}}, + } ) - assert proc.returncode == 0, proc.stderr.decode() - calls = load_proxy_sidecar_calls(sidecar) - assert len(calls) == 1 - assert calls[0]["tool"] == "ping" - - def test_proxy_child_env_pythonpath_clean(tmp_path): - server = tmp_path / "check_env.py" - server.write_text( - textwrap.dedent( - f""" + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), # foreign cwd — must still import evals.proxy + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "ping" + + +def _proxy_child_env_pythonpath_clean(tmp_path): + server = tmp_path / "check_env.py" + server.write_text( + textwrap.dedent( + f""" import json, os, sys root = {str(REPO)!r} pp = os.environ.get("PYTHONPATH", "") @@ -362,54 +368,55 @@ def test_proxy_child_env_pythonpath_clean(tmp_path): sys.stdout.flush() sys.exit(0 if not bad else 9) """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - foreign = tmp_path / "foreign" - foreign.mkdir() - env = ensure_proxy_pythonpath(dict(__import__("os").environ)) - assert str(REPO) in env.get("PYTHONPATH", "") - client_in = ( - json.dumps( - { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "envcheck", "arguments": {}}, - } - ) - + "\n" - ) - proc = subprocess.run( - [ - sys.executable, - "-m", - "evals.proxy", - "--log", - str(sidecar), - "--", - sys.executable, - str(server), - ], - input=client_in.encode(), - capture_output=True, - cwd=str(foreign), - env=env, - timeout=15, + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(__import__("os").environ)) + assert str(REPO) in env.get("PYTHONPATH", "") + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "envcheck", "arguments": {}}, + } ) - assert proc.returncode == 0, proc.stderr.decode() - assert b"bad=False" in proc.stdout + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + assert b"bad=False" in proc.stdout - def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path): - import os - import signal - import time - server = tmp_path / "echo_server.py" - server.write_text( - textwrap.dedent( - """ +def _proxy_survives_cli_group_kill_and_writes_meta(tmp_path): + import os + import signal + import time + + server = tmp_path / "echo_server.py" + server.write_text( + textwrap.dedent( + """ import json, sys for line in sys.stdin: line = line.strip() @@ -424,14 +431,14 @@ def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path): sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n") sys.stdout.flush() """ - ), - encoding="utf-8", - ) - sidecar = tmp_path / "side.jsonl" - leader_script = tmp_path / "cli_leader.py" - leader_script.write_text( - textwrap.dedent( - f""" + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + leader_script = tmp_path / "cli_leader.py" + leader_script.write_text( + textwrap.dedent( + f""" import os, subprocess, sys, time from pathlib import Path sidecar = Path({str(sidecar)!r}) @@ -461,142 +468,152 @@ def test_proxy_survives_cli_group_kill_and_writes_meta(tmp_path): # Stay alive as group leader until killed by the test harness. time.sleep(9999) """ - ), - encoding="utf-8", - ) + ), + encoding="utf-8", + ) - # Leader is a process-group leader (like run_cli_subprocess). - env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} - leader = subprocess.Popen( - [sys.executable, str(leader_script)], - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - cwd=str(REPO), - env=env, - ) + # Leader is a process-group leader (like run_cli_subprocess). + env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} + leader = subprocess.Popen( + [sys.executable, str(leader_script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(REPO), + env=env, + ) + try: + # Wait until proxy has started (sidecar created) and setsid likely done. + boot = time.monotonic() + 5.0 + while time.monotonic() < boot: + if sidecar.is_file(): + break + time.sleep(0.05) + time.sleep(0.5) # allow setsid + optional tools/call + # SIGKILL the CLI process group — must NOT kill the detached proxy. try: - # Wait until proxy has started (sidecar created) and setsid likely done. - boot = time.monotonic() + 5.0 - while time.monotonic() < boot: - if sidecar.is_file(): + os.killpg(leader.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + leader.kill() + leader.wait(timeout=1.0) + + # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. + deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 + meta_seen = False + while time.monotonic() < deadline: + if sidecar.is_file(): + text = sidecar.read_text(encoding="utf-8") + if "proxy_meta" in text: + meta_seen = True break - time.sleep(0.05) - time.sleep(0.5) # allow setsid + optional tools/call - # SIGKILL the CLI process group — must NOT kill the detached proxy. + time.sleep(0.1) + assert meta_seen, ( + f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" + ) + rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + finally: + if leader.poll() is None: try: os.killpg(leader.pid, signal.SIGKILL) - except ProcessLookupError: - pass + except Exception: + leader.kill() try: leader.wait(timeout=2.0) - except subprocess.TimeoutExpired: - leader.kill() - leader.wait(timeout=1.0) - - # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. - deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 - meta_seen = False - while time.monotonic() < deadline: - if sidecar.is_file(): - text = sidecar.read_text(encoding="utf-8") - if "proxy_meta" in text: - meta_seen = True - break - time.sleep(0.1) - assert meta_seen, ( - f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" - ) - rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] - assert rows[-1].get("row_type") == "proxy_meta" - finally: - if leader.poll() is None: - try: - os.killpg(leader.pid, signal.SIGKILL) - except Exception: - leader.kill() - try: - leader.wait(timeout=2.0) - except Exception: - pass + except Exception: + pass - _d0 = tmp_path / "test_proxy_records_tools_call_and_exit_code" - _d0.mkdir() - test_proxy_records_tools_call_and_exit_code(_d0) - _d1 = tmp_path / "test_proxy_byte_faithful_child_receives_exact_bytes" - _d1.mkdir() - test_proxy_byte_faithful_child_receives_exact_bytes(_d1) - test_proxy_main_requires_command() - _d3 = tmp_path / "test_proxy_exits_when_child_dies_first" - _d3.mkdir() - test_proxy_exits_when_child_dies_first(_d3) - _d4 = tmp_path / "test_proxy_from_foreign_cwd_with_pythonpath" - _d4.mkdir() - test_proxy_from_foreign_cwd_with_pythonpath(_d4) - _d5 = tmp_path / "test_proxy_child_env_pythonpath_clean" - _d5.mkdir() - test_proxy_child_env_pythonpath_clean(_d5) - _d6 = tmp_path / "test_proxy_survives_cli_group_kill_and_writes_meta" - _d6.mkdir() - test_proxy_survives_cli_group_kill_and_writes_meta(_d6) - - -def test_sidecar_behaviours(tmp_path): - def test_sidecar_recorder_unit(tmp_path): - rec = SidecarRecorder(tmp_path / "a.jsonl") - rec.on_client_message( - { - "jsonrpc": "2.0", - "id": 9, - "method": "tools/call", - "params": {"name": "t", "arguments": {"a": 1}}, - } - ) - rec.on_server_message({"jsonrpc": "2.0", "id": 9, "result": {"content": [], "isError": False}}) - rec.write_meta() - calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") - raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] - raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") - assert len(calls) == 1 - assert calls[0]["tool"] == "t" - assert calls[0]["args"] == {"a": 1} - assert calls[0]["origin"] == "plane" - assert "result_text" not in calls[0] - assert "result_text" not in raw_call - assert rec.finalized is True - - def test_sidecar_result_payload_round_trips_only_when_enabled(tmp_path): - path = tmp_path / "payload.jsonl" - rec = SidecarRecorder(path, record_result_payloads=True) - rec.on_client_message( - { - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": {"name": "find_work_items", "arguments": {}}, - } - ) - result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} - rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) - rec.write_meta() - - expected_text = json.dumps(result, default=str, ensure_ascii=False) - raw_call = next( - row - for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) - if row.get("row_type") != "proxy_meta" - ) - assert raw_call["result_text"] == expected_text - calls = load_proxy_sidecar_calls(path) - assert calls[0]["result_text"] == expected_text - assert calls[0]["result_chars"] == len(expected_text) - _d0 = tmp_path / "test_sidecar_recorder_unit" - _d0.mkdir() - test_sidecar_recorder_unit(_d0) - _d1 = tmp_path / "test_sidecar_result_payload_round_trips_only_when_enabled" - _d1.mkdir() - test_sidecar_result_payload_round_trips_only_when_enabled(_d1) +@pytest.mark.parametrize( + "case", + case_params( + _proxy_records_tools_call_and_exit_code, + _proxy_byte_faithful_child_receives_exact_bytes, + _proxy_main_requires_command, + _proxy_exits_when_child_dies_first, + _proxy_from_foreign_cwd_with_pythonpath, + _proxy_child_env_pythonpath_clean, + _proxy_survives_cli_group_kill_and_writes_meta, + ), +) +def test_proxy_behaviours(case, tmp_path): + case(tmp_path) + + +def _sidecar_recorder_unit(tmp_path): + sentinel = "hidden-target-fact-7b0a1f9c" + rec = SidecarRecorder( + tmp_path / "a.jsonl", + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": {"name": "t", "arguments": {"a": 1}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 9, + "result": {"content": [{"type": "text", "text": f"target={sentinel}"}], "isError": False}, + } + ) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") + raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] + raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") + assert len(calls) == 1 + assert calls[0]["tool"] == "t" + assert calls[0]["args"] == {"a": 1} + assert calls[0]["origin"] == "plane" + assert "result_text" not in calls[0] + assert "result_text" not in raw_call + assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert raw_call["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert sentinel not in (tmp_path / "a.jsonl").read_text(encoding="utf-8") + assert rec.finalized is True + + +def _sidecar_result_payload_round_trips_only_when_enabled(tmp_path): + path = tmp_path / "payload.jsonl" + rec = SidecarRecorder(path, record_result_payloads=True) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "find_work_items", "arguments": {}}, + } + ) + result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} + rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) + rec.write_meta() + + expected_text = json.dumps(result, default=str, ensure_ascii=False) + raw_call = next( + row + for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) + if row.get("row_type") != "proxy_meta" + ) + assert raw_call["result_text"] == expected_text + calls = load_proxy_sidecar_calls(path) + assert calls[0]["result_text"] == expected_text + assert calls[0]["result_chars"] == len(expected_text) + + +@pytest.mark.parametrize( + "case", + case_params(_sidecar_recorder_unit, _sidecar_result_payload_round_trips_only_when_enabled), +) +def test_sidecar_behaviours(case, tmp_path): + case(tmp_path) def test_append_after_finalize_is_dropped(tmp_path: Path): @@ -838,10 +855,15 @@ def test_child_exit_drains_final_response(tmp_path: Path): def test_scrub_child_pythonpath_removes_repo(): import os - env = {"PYTHONPATH": f"{REPO}{os.pathsep}/other/lib", "FOO": "1"} + env = { + "PYTHONPATH": f"{REPO}{os.pathsep}/other/lib", + "FOO": "1", + EVIDENCE_SENTINELS_ENV: '{"target":["secret"]}', + } scrubbed = scrub_child_pythonpath(env) assert "/other/lib" in scrubbed["PYTHONPATH"] assert str(REPO) not in scrubbed["PYTHONPATH"].split(os.pathsep) + assert EVIDENCE_SENTINELS_ENV not in scrubbed # Only-repo entry drops the var entirely only = scrub_child_pythonpath({"PYTHONPATH": str(REPO)}) assert "PYTHONPATH" not in only diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py index 9d3708fb..b2c673d0 100644 --- a/tests/evals/test_results.py +++ b/tests/evals/test_results.py @@ -4,9 +4,12 @@ from collections import deque from contextlib import asynccontextmanager +from dataclasses import fields from types import SimpleNamespace from typing import Any +import pytest + from evals.drivers import ( ApiDriver, ) @@ -17,12 +20,23 @@ ToolSpec, Turn, ) -from evals.results import RESULT_SCHEMA_VERSION, AgentRun, CallRecord, TaskResult, Usage, agent_run_to_harness_dict -from evals.runner.live import classify_call +from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.results import ( + AGENT_RESULT_COPY_FIELDS, + AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS, + RESULT_SCHEMA_VERSION, + TASK_RESULT_HARNESS_FIELDS, + AgentRun, + CallRecord, + TaskResult, + Usage, + agent_run_to_harness_dict, +) from evals.token_counting import estimate_result_tokens from evals.tool_names import ( normalize_tool_call, ) +from tests.evals.conftest import case_params class FakeBackend: @@ -103,7 +117,7 @@ def run_driver(driver: ApiDriver, *, max_turns: int = 5): ) -def test_api_driver_maps_every_legacy_row_field(): +def test_api_driver_maps_every_current_row_field(): backend = FakeBackend( [ Turn( @@ -122,22 +136,13 @@ def test_api_driver_maps_every_legacy_row_field(): ] ) run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) - row = agent_run_to_harness_dict( - run, - optimal={"lookup"}, - alternate=set(), - classify=lambda tool, optimal, alternate: ( - "optimal" if tool in optimal else "alternate" if tool in alternate else "out_of_set" - ), - ) + row = agent_run_to_harness_dict(run) required = { "final_text", "calls", "num_calls", "errored_calls", - "alternate_calls", - "out_of_set_calls", "total_result_tokens", "usage_per_iteration", "cum_input_tokens", @@ -151,7 +156,6 @@ def test_api_driver_maps_every_legacy_row_field(): assert required <= row.keys() assert { "tool", - "class", "args_chars", "result_tokens", "result_chars", @@ -167,162 +171,155 @@ def test_api_driver_maps_every_legacy_row_field(): assert row["provider_stop_reason"] == "fake_done" -def test_agent_run_behaviours(): - def test_agent_run_dict_keeps_action_arg(): - run = AgentRun( - calls=[ - {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, - {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, - ], - final_text="done", - usage=None, - stopped_reason="end_turn", - ) - d = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=lambda t, o, a: "out_of_set", - ) - assert d["calls"][0]["action"] == "create" - assert "action" not in d["calls"][1] - - def test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks(): - run = AgentRun( - calls=[ - normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), - ], - client_tool_calls=[ - normalize_tool_call("ToolSearch", {"query": "work items"}), - ], - final_text="done", - usage={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_cost_usd": 0.29, - "modelUsage": { - "claude-sonnet": { - "inputTokens": 10, - "outputTokens": 865, - "cacheReadInputTokens": 250433, - "cacheCreationInputTokens": 33838, - "costUSD": 0.29, - } - }, - }, - usage_total={ - "input_tokens": 10, - "output_tokens": 865, - "cache_read_input_tokens": 250433, - "cache_creation_input_tokens": 33838, - "total_input_tokens_including_cache": 10 + 250433 + 33838, - "total_cost_usd": 0.29, - "source": "modelUsage", - }, - stopped_reason="end_turn", - usage_scope="run", - call_source="transcript", - hit_max_turns=False, - wall_time_s=1.5, - ) - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate={"get_work_item"}, - classify=classify_call, - ) - assert out["num_calls"] == 1 - assert out["out_of_set_calls"] == 0 - assert out["calls"][0]["class"] == "optimal" - assert out["client_tool_call_count"] == 1 - assert out["client_tool_calls"][0]["tool"] == "ToolSearch" - # F2: cum_input_tokens null — not the misleading uncached-only 10 - assert out["cum_input_tokens"] is None - assert out["cum_input_tokens_reason"] - assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 - assert out["usage_per_iteration"] == [] - assert out["calls"][0]["result_tokens"] == 0 - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True - assert "result_tokens_skipped_reason" not in out - - def test_agent_run_hit_max_maps_to_hit_max_iterations(): - run = AgentRun( - calls=[], - final_text="", - usage=None, - stopped_reason="end_turn", - hit_max_turns=True, - call_source="json", - ) - out = agent_run_to_harness_dict(run, optimal=set(), alternate=set(), classify=classify_call) - assert out["hit_max_iterations"] is True - assert out["stop_reason"] == "max_turns" - - def test_agent_run_to_harness_dict_does_not_guess_usage_total(): - run = AgentRun( - calls=[], - final_text="ok", - usage={ - "input_tokens": 5000, - "output_tokens": 200, - # Codex-ish shape — not Claude modelUsage. A Claude rebuild would - # silently produce a wrong / empty total if reintroduced. - "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, - }, - usage_total=None, - stopped_reason="completed", - usage_scope="run", - call_source="stream", - ) - out = agent_run_to_harness_dict( - run, - optimal=set(), - alternate=set(), - classify=classify_call, - ) - assert out["usage"] == run.usage - assert out["usage_total"] is None - - def test_agent_run_to_harness_propagates_proxy_fields(): - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {"q": "a"}, - "origin": "plane", - "is_error": True, - "result_chars": 99, - "duration_ms": 42, +def _agent_run_dict_keeps_action_arg(): + run = AgentRun( + calls=[ + {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, + {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, + ], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + d = agent_run_to_harness_dict(run) + assert d["calls"][0]["action"] == "create" + assert "action" not in d["calls"][1] + + +def _agent_run_to_harness_dict_excludes_toolsearch_from_plane_calls(): + run = AgentRun( + calls=[ + normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), + ], + client_tool_calls=[ + normalize_tool_call("ToolSearch", {"query": "work items"}), + ], + final_text="done", + usage={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_cost_usd": 0.29, + "modelUsage": { + "claude-sonnet": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.29, } - ], - final_text="x", - usage=None, - stopped_reason="end_turn", - call_source="proxy", - usage_scope="run", - ) - d = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=lambda t, o, a: "optimal", - ) - assert d["calls"][0]["is_error"] is True - assert d["calls"][0]["result_chars"] == 99 - assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) - assert d["calls"][0]["result_tokens_estimated"] is True - assert d["result_tokens_estimated"] is True - assert d["calls"][0]["duration_ms"] == 42 - assert d["errored_calls"] == 1 + }, + }, + usage_total={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_input_tokens_including_cache": 10 + 250433 + 33838, + "total_cost_usd": 0.29, + "source": "modelUsage", + }, + stopped_reason="end_turn", + usage_scope="run", + call_source="transcript", + hit_max_turns=False, + wall_time_s=1.5, + ) + out = agent_run_to_harness_dict(run) + assert out["num_calls"] == 1 + assert out["client_tool_call_count"] == 1 + assert out["client_tool_calls"][0]["tool"] == "ToolSearch" + # F2: cum_input_tokens null — not the misleading uncached-only 10 + assert out["cum_input_tokens"] is None + assert out["cum_input_tokens_reason"] + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["usage_per_iteration"] == [] + assert out["calls"][0]["result_tokens"] == 0 + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + assert "result_tokens_skipped_reason" not in out + + +def _agent_run_hit_max_maps_to_hit_max_iterations(): + run = AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + hit_max_turns=True, + call_source="json", + ) + out = agent_run_to_harness_dict(run) + assert out["hit_max_iterations"] is True + assert out["stop_reason"] == "max_turns" + + +def _agent_run_to_harness_dict_does_not_guess_usage_total(): + run = AgentRun( + calls=[], + final_text="ok", + usage={ + "input_tokens": 5000, + "output_tokens": 200, + # Codex-ish shape — not Claude modelUsage. A Claude rebuild would + # silently produce a wrong / empty total if reintroduced. + "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, + }, + usage_total=None, + stopped_reason="completed", + usage_scope="run", + call_source="stream", + ) + out = agent_run_to_harness_dict(run) + assert out["usage"] == run.usage + assert out["usage_total"] is None + - test_agent_run_dict_keeps_action_arg() - test_agent_run_to_harness_dict_excludes_toolsearch_from_mispicks() - test_agent_run_hit_max_maps_to_hit_max_iterations() - test_agent_run_to_harness_dict_does_not_guess_usage_total() - test_agent_run_to_harness_propagates_proxy_fields() +def _agent_run_to_harness_propagates_proxy_fields(): + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {"q": "a"}, + "origin": "plane", + "is_error": True, + "result_chars": 99, + "duration_ms": 42, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + final_text="x", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + usage_scope="run", + evidence_trace_available=True, + ) + d = agent_run_to_harness_dict(run) + assert d["calls"][0]["is_error"] is True + assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) + assert d["calls"][0]["result_tokens_estimated"] is True + assert d["result_tokens_estimated"] is True + assert d["calls"][0]["duration_ms"] == 42 + assert d["calls"][0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert d["evidence_trace_available"] is True + assert d["errored_calls"] == 1 + + +@pytest.mark.parametrize( + "case", + case_params( + _agent_run_dict_keeps_action_arg, + _agent_run_to_harness_dict_excludes_toolsearch_from_plane_calls, + _agent_run_hit_max_maps_to_hit_max_iterations, + _agent_run_to_harness_dict_does_not_guess_usage_total, + _agent_run_to_harness_propagates_proxy_fields, + ), +) +def test_agent_run_behaviours(case): + case() def test_task_result_schema_round_trip_owns_usage_shape(): @@ -331,16 +328,19 @@ def test_task_result_schema_round_trip_owns_usage_shape(): task_id="R1", label="local", server="local", + expected_rows=35, + cleanup_error="RuntimeError: teardown failed", calls=[ CallRecord( tool="find_work_items", - classification="optimal", result_tokens=3, result_tokens_estimated=False, result_token_count_method="backend", + observed_sentinels=[TARGET_ENTITY_EVIDENCE], ) ], num_calls=1, + evidence_trace_available=True, usage_per_iteration=[Usage(10, 2, 3, 4)], ) @@ -349,8 +349,31 @@ def test_task_result_schema_round_trip_owns_usage_shape(): assert row["row_type"] == "result" assert row["label"] == "local" assert row["server"] == "local" + assert row["expected_rows"] == 35 + assert row["cleanup_error"] == "RuntimeError: teardown failed" assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] loaded = TaskResult.from_row(row) assert loaded.row_type == "result" assert loaded.calls[0].tool == "find_work_items" + assert loaded.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] + assert loaded.evidence_trace_available is True + assert loaded.expected_rows == 35 + assert loaded.cleanup_error == "RuntimeError: teardown failed" assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] + + +def test_apply_agent_result_reflection_parity_and_skipped_reason_copy(): + declared = {field.name for field in fields(TaskResult)} + copied = set(AGENT_RESULT_COPY_FIELDS) + optional_identity = set(AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS) + harness_owned = set(TASK_RESULT_HARNESS_FIELDS) + + assert not (copied & optional_identity or copied & harness_owned or optional_identity & harness_owned) + assert declared == copied | optional_identity | harness_owned + + row = TaskResult(task_id="R1", result_tokens_skipped_reason=None) + agent = TaskResult(task_id="must-not-replace", result_tokens_skipped_reason="payload recording disabled") + row.apply_agent_result(agent) + + assert row.task_id == "R1" + assert row.result_tokens_skipped_reason == "payload recording disabled" diff --git a/tests/evals/test_skip_taxonomy.py b/tests/evals/test_skip_taxonomy.py new file mode 100644 index 00000000..42b7df42 --- /dev/null +++ b/tests/evals/test_skip_taxonomy.py @@ -0,0 +1,54 @@ +"""Offline tests for the explicit environment skip taxonomy.""" + +from __future__ import annotations + +import pytest + +from evals.skip_taxonomy import ( + PLAN_GATED_CAPABILITIES, + classify_skip_reason, + is_expected_environment_capability_skip, + skip_reason_family, +) + + +@pytest.mark.parametrize( + ("reason", "disposition", "family"), + [ + pytest.param("env:plan-gated:customers", "expected-capability", "plan-gated", id="plan-gated"), + pytest.param("env:no-activity-worker", "expected-capability", "no-activity-worker", id="activity-worker"), + pytest.param( + "env:no-activity-worker (ConnectionError: unavailable)", + "unexpected", + "env:no-activity-worker (ConnectionError: unavailable)", + id="activity-worker-detail-is-not-a-capability-skip", + ), + pytest.param( + "env:fixture-collision:customers:Acme", + "dirty-environment", + "fixture-collision", + id="fixture-collision", + ), + pytest.param("env:new-capability", "unexpected", "env:new-capability", id="unknown-env-reason"), + pytest.param( + "env:plan-gated:customerz", + "unexpected", + "env:plan-gated:customerz", + id="unknown-plan-gated-capability", + ), + pytest.param("env:plan-gated:", "unexpected", "env:plan-gated:", id="malformed-plan-gate"), + pytest.param("env:no-activity-worker-new", "unexpected", "env:no-activity-worker-new", id="near-miss"), + ], +) +def test_skip_reason_taxonomy_is_explicit_and_fail_closed(reason, disposition, family): + assert classify_skip_reason(reason) == disposition + assert is_expected_environment_capability_skip(reason) is (disposition == "expected-capability") + assert skip_reason_family(reason) == family + + +def test_plan_gated_capability_allowlist_matches_reviewed_seed_surfaces(): + assert PLAN_GATED_CAPABILITIES == frozenset( + {"customers", "releases", "work-item-types", "initiatives", "teamspaces"} + ) + for capability in PLAN_GATED_CAPABILITIES: + assert classify_skip_reason(f"env:plan-gated:{capability}") == "expected-capability", capability diff --git a/tests/evals/test_token_counting.py b/tests/evals/test_token_counting.py index 650745b1..fca9b34a 100644 --- a/tests/evals/test_token_counting.py +++ b/tests/evals/test_token_counting.py @@ -7,15 +7,18 @@ import pytest from evals.results import AgentRun, agent_run_to_harness_dict -from evals.runner.live import classify_call from evals.token_counting import estimate_result_tokens -def test_agent_behaviours(monkeypatch): - def test_agent_run_payload_uses_importable_tokenizer(monkeypatch): +@pytest.mark.parametrize("has_tokenizer", [True, False], ids=["importable-tokenizer", "estimator-fallback"]) +def test_agent_behaviours(monkeypatch, has_tokenizer): + text = "serialized workspace result" + + if has_tokenizer: + class FakeEncoding: - def encode(self, text): - assert text == "serialized workspace result" + def encode(self, encoded_text): + assert encoded_text == text return [10, 20, 30] class FakeTiktoken: @@ -25,69 +28,33 @@ def get_encoding(name): return FakeEncoding() monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len("serialized workspace result"), - "result_text": "serialized workspace result", - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", - ) - - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, - ) + else: + monkeypatch.setitem(sys.modules, "tiktoken", None) - assert out["calls"][0]["result_tokens"] == 3 - assert out["calls"][0]["result_tokens_estimated"] is False + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len(text), + "result_text": text, + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict(run) + + expected_tokens = 3 if has_tokenizer else estimate_result_tokens(len(text)) + assert out["calls"][0]["result_tokens"] == expected_tokens + assert out["calls"][0]["result_tokens_estimated"] is not has_tokenizer + assert out["result_tokens_estimated"] is not has_tokenizer + assert "result_text" not in out["calls"][0] + if has_tokenizer: assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" - assert out["result_tokens_estimated"] is False assert out["result_tokens_mode"] == "measured" - assert "result_text" not in out["calls"][0] - - def test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(monkeypatch): - monkeypatch.setitem(sys.modules, "tiktoken", None) - text = "payload without a tokenizer" - run = AgentRun( - calls=[ - { - "tool": "find_work_items", - "args": {}, - "origin": "plane", - "result_chars": len(text), - "result_text": text, - } - ], - final_text="ok", - usage=None, - stopped_reason="completed", - usage_scope="run", - call_source="proxy", - ) - - out = agent_run_to_harness_dict( - run, - optimal={"find_work_items"}, - alternate=set(), - classify=classify_call, - ) - - assert out["calls"][0]["result_tokens"] == estimate_result_tokens(len(text)) - assert out["calls"][0]["result_tokens_estimated"] is True - assert out["result_tokens_estimated"] is True - - with pytest.MonkeyPatch.context() as mp: - test_agent_run_payload_uses_importable_tokenizer(mp) - with pytest.MonkeyPatch.context() as mp: - test_agent_run_payload_falls_back_to_shared_estimator_without_tokenizer(mp) From 12933c760d689d63d9f494f6257fd2dbcfe343f2 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 20:56:42 +0530 Subject: [PATCH 37/93] Bind read provenance to the target entity, and stop shipping the secret Provenance accepted an API-confirmed value appearing in any successful response, so an agent could recover the value, write it onto an unrelated entity, read that back, and satisfy provenance without ever touching the seeded fact. A match now requires both the value and request arguments targeting the seeded entity id. No tool name is asserted, and response bodies are still never persisted. The value itself no longer leaves the harness. Every CLI surface leaked it - codex through argv, claude through its referenced MCP JSON, opencode through its working-directory config, antigravity through its fake-HOME configs. The proxy now reads a mode-0600 file outside the agent's cwd, unlinked when consumed, carrying target ids and one-way (length, sha256) fingerprints rather than the values. Following the pathname yields nothing invertible. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/__init__.py | 3 +- evals/drivers/cli/sidecar.py | 195 ++++++++++++++++++++++--- evals/drivers/driver.py | 133 ++++++++++++++--- evals/evidence.py | 268 ++++++++++++++++++++++++++++++++--- 4 files changed, 539 insertions(+), 60 deletions(-) diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index c21173a9..d263ba32 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -42,7 +42,7 @@ proxy_wrap_server_command, wait_for_proxy_meta, ) -from evals.drivers.driver import ApiDriver, CliDriver +from evals.drivers.driver import ApiDriver, CliDriver, CliRunError # Registry # --------------------------------------------------------------------------- @@ -72,6 +72,7 @@ def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: "ApiDriver", "ClaudeCliDriver", "CliDriver", + "CliRunError", "CodexCliDriver", "OpencodeCliDriver", "apply_proxy_sidecar", diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index dabec415..56439a53 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -6,10 +6,38 @@ import os import sys import time +from collections.abc import Iterator +from dataclasses import dataclass from pathlib import Path from typing import Any from evals import REPO_ROOT +from evals.results import TraceIntegrityReason + + +@dataclass(slots=True) +class ProxySidecarResult: + """Harvested calls plus typed integrity and manifest observations.""" + + calls: list[dict[str, Any]] + client_calls: list[dict[str, Any]] + call_source: str + trace_integrity: bool + trace_integrity_reason: TraceIntegrityReason | None + tool_manifest_fingerprint: str | None + status: dict[str, Any] + + def __iter__(self) -> Iterator[Any]: + """Retain the established three-value unpacking API.""" + yield self.calls + yield self.client_calls + yield self.call_source + + +def _nonnegative_int(value: Any) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value def proxy_wrap_server_command( @@ -55,7 +83,9 @@ def load_proxy_sidecar( - torn_line: final line failed to parse - skipped_rows: non-final rows that could not produce a call or metadata row - meta: proxy_meta row if present - - pending_left: from meta when present + - pending_left / non_tool_pending_left: unmatched requests from meta + - sequence_errors: invalid, duplicate, missing, or unexpected call sequence values + - proxy_meta_count / proxy_meta_not_final: metadata framing integrity """ status: dict[str, Any] = { "state": "missing", @@ -63,6 +93,14 @@ def load_proxy_sidecar( "skipped_rows": 0, "meta": None, "pending_left": None, + "non_tool_pending_left": None, + "proxy_meta_count": 0, + "proxy_meta_not_final": False, + "invalid_seq": 0, + "duplicate_seq": 0, + "missing_seq": 0, + "unexpected_seq": 0, + "invalid_meta_fields": 0, } if not path.is_file(): return [], status @@ -79,12 +117,15 @@ def load_proxy_sidecar( lines = text.splitlines() calls: list[dict[str, Any]] = [] meta: dict[str, Any] | None = None + meta_positions: list[int] = [] + nonblank_position = 0 torn = False skipped_rows = 0 for i, line in enumerate(lines): s = line.strip() if not s: continue + nonblank_position += 1 try: row = json.loads(s) except json.JSONDecodeError: @@ -99,6 +140,7 @@ def load_proxy_sidecar( continue if row.get("row_type") == "proxy_meta": meta = row + meta_positions.append(nonblank_position) continue tool = row.get("tool") if not tool: @@ -120,20 +162,76 @@ def load_proxy_sidecar( call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] calls.append(call) - # Score order must match request seq, not response-append order. - calls.sort(key=lambda c: (c.get("seq") is None, c.get("seq") if c.get("seq") is not None else 0)) + valid_sequences = [call["seq"] for call in calls if _nonnegative_int(call.get("seq")) not in (None, 0)] + invalid_seq = len(calls) - len(valid_sequences) + duplicate_seq = len(valid_sequences) - len(set(valid_sequences)) + last_seq = _nonnegative_int(meta.get("last_seq")) if meta is not None else None + tool_request_count = _nonnegative_int(meta.get("tool_request_count")) if meta is not None else None + invalid_meta_fields = int(meta is not None and last_seq is None) + int( + meta is not None and tool_request_count is None + ) + if last_seq is not None and tool_request_count is not None and tool_request_count != last_seq: + invalid_meta_fields += 1 + expected_sequences = set(range(1, last_seq + 1)) if last_seq is not None else set() + observed_sequences = set(valid_sequences) + missing_seq = len(expected_sequences - observed_sequences) + unexpected_seq = len(observed_sequences - expected_sequences) if last_seq is not None else 0 + + # Score order must match request seq, not response-append order. Invalid seq + # rows remain available for diagnostics but can never make the trace valid. + calls.sort( + key=lambda call: ( + _nonnegative_int(call.get("seq")) in (None, 0), + _nonnegative_int(call.get("seq")) or 0, + ) + ) status["torn_line"] = torn status["skipped_rows"] = skipped_rows status["meta"] = meta + status["proxy_meta_count"] = len(meta_positions) + status["proxy_meta_not_final"] = bool(meta_positions and meta_positions[-1] != nonblank_position) + status["invalid_seq"] = invalid_seq + status["duplicate_seq"] = duplicate_seq + status["missing_seq"] = missing_seq + status["unexpected_seq"] = unexpected_seq + status["invalid_meta_fields"] = invalid_meta_fields if meta is not None: - status["pending_left"] = meta.get("pending_left") + counter_keys = ( + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "non_json_lines", + "malformed_jsonrpc", + "recorder_errors", + ) + for key in counter_keys: + value = _nonnegative_int(meta.get(key)) + if meta.get(key) is not None and value is None: + status["invalid_meta_fields"] += 1 + status[key] = value status["pumps_alive"] = bool(meta.get("pumps_alive")) + fingerprint = meta.get("tool_manifest_fingerprint") + status["tool_manifest_fingerprint"] = str(fingerprint) if isinstance(fingerprint, str) else None + fatal_counts = ( + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "recorder_errors", + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ) incomplete = bool( torn or skipped_rows > 0 - or meta is None - or (meta is not None and int(meta.get("pending_left") or 0) > 0) + or len(meta_positions) != 1 + or status["proxy_meta_not_final"] + or any((status.get(key) or 0) > 0 for key in fatal_counts) or (meta is not None and bool(meta.get("pumps_alive"))) ) if not calls and not meta and not torn and skipped_rows == 0: @@ -145,6 +243,48 @@ def load_proxy_sidecar( return calls, status +def trace_integrity_from_status( + status: dict[str, Any], +) -> tuple[bool, TraceIntegrityReason | None]: + """Map sidecar status to the typed result-row integrity fields.""" + if status.get("state") == "complete": + return True, None + if (status.get("unparsed_lines") or 0) > 0: + return False, "protocol_violation" + return False, "recorder_loss" + + +def _incompleteness_note(status: dict[str, Any]) -> str: + parts = ["proxy_sidecar_incomplete"] + if status.get("torn_line"): + parts.append("torn_line=1") + for key in ( + "skipped_rows", + "proxy_meta_count", + "proxy_meta_not_final", + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "non_json_lines", + "malformed_jsonrpc", + "recorder_errors", + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ): + value = status.get(key) + if value and not (key == "proxy_meta_count" and value == 1): + parts.append(f"{key}={int(value)}") + if status.get("meta") is None: + parts.append("no_meta=1") + if status.get("pumps_alive"): + parts.append("pumps_alive=1") + return ":".join(parts) + + def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: """Convenience: call rows only (sorted by seq).""" calls, _status = load_proxy_sidecar(path) @@ -156,7 +296,7 @@ def apply_proxy_sidecar( client_calls: list[dict[str, Any]], sidecar_path: Path, notes: list[str], -) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: +) -> ProxySidecarResult: """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. Incomplete sidecar (torn/skipped row, missing meta, pending_left>0) yields @@ -165,28 +305,39 @@ def apply_proxy_sidecar( """ proxy_calls, status = load_proxy_sidecar(sidecar_path) state = status.get("state") + trace_integrity, trace_integrity_reason = trace_integrity_from_status(status) + fingerprint = status.get("tool_manifest_fingerprint") if trace_integrity else None + + def result( + selected_calls: list[dict[str, Any]], + selected_client_calls: list[dict[str, Any]], + source: str, + ) -> ProxySidecarResult: + return ProxySidecarResult( + calls=selected_calls, + client_calls=selected_client_calls, + call_source=source, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=str(fingerprint) if isinstance(fingerprint, str) else None, + status=status, + ) + if state in ("missing", "empty"): notes.append("proxy_sidecar_empty") - return calls, client_calls, "json" + return result(calls, client_calls, "json") if state == "incomplete": - notes.append( - "proxy_sidecar_incomplete" - + (":torn" if status.get("torn_line") else "") - + (f":skipped_rows={status.get('skipped_rows')}" if status.get("skipped_rows") else "") - + (":no_meta" if status.get("meta") is None else "") - + (f":pending_left={status.get('pending_left')}" if status.get("pending_left") else "") - + (":pumps_alive" if status.get("pumps_alive") else "") - ) + notes.append(_incompleteness_note(status)) if len(calls) > len(proxy_calls): notes.append("proxy_sidecar_deferred_to_cli_trace") - return calls, client_calls, "json" + return result(calls, client_calls, "json") if proxy_calls: notes.append(f"calls_from_proxy:{sidecar_path}") - return proxy_calls, client_calls, "proxy" - return calls, client_calls, "json" + return result(proxy_calls, client_calls, "proxy") + return result(calls, client_calls, "json") # complete notes.append(f"calls_from_proxy:{sidecar_path}") - return proxy_calls, client_calls, "proxy" + return result(proxy_calls, client_calls, "proxy") def wait_for_proxy_meta( @@ -226,7 +377,7 @@ def harvest_proxy_after_cli_timeout( notes: list[str], *, max_wait_s: float | None = None, -) -> tuple[list[dict[str, Any]], list[dict[str, Any]], str]: +) -> ProxySidecarResult: """Wait for proxy finalization after CLI kill, then harvest the sidecar. If meta never appears within the wait window, harvest anyway (incomplete @@ -246,5 +397,7 @@ def harvest_proxy_after_cli_timeout( "load_proxy_sidecar", "load_proxy_sidecar_calls", "proxy_wrap_server_command", + "ProxySidecarResult", + "trace_integrity_from_status", "wait_for_proxy_meta", ] diff --git a/evals/drivers/driver.py b/evals/drivers/driver.py index 7ecfab8f..85418e64 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/driver.py @@ -30,6 +30,7 @@ ) from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess from evals.drivers.cli.sidecar import ( + ProxySidecarResult, apply_proxy_sidecar, ensure_proxy_pythonpath, harvest_proxy_after_cli_timeout, @@ -38,13 +39,16 @@ ) from evals.evidence import ( configured_evidence_labels, + normalize_evidence_aggregates, normalize_evidence_sentinels, normalize_evidence_targets, + observed_aggregate_labels, observed_sentinel_labels, write_evidence_config, ) from evals.results import AgentRun, Usage from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens +from evals.tool_manifest import ToolManifestCapture, tools_page DEFAULT_MAX_TOKENS = 8192 @@ -172,7 +176,12 @@ def _server_params(self, mcp_env: dict[str, str], cwd: Path | None) -> StdioServ ) @asynccontextmanager - async def _mcp_session(self, params: StdioServerParameters): + async def _mcp_session( + self, + params: StdioServerParameters, + *, + manifest_state: dict[str, bool], + ): if self.mcp_session_factory is not None: context = self.mcp_session_factory(params) if inspect.isawaitable(context): @@ -185,9 +194,34 @@ async def _mcp_session(self, params: StdioServerParameters): return async with stdio_client(params) as (read, write): - async with ClientSession(read, write) as session: + + async def message_handler(message: Any) -> None: + notification = getattr(message, "root", message) + if getattr(notification, "method", None) == "notifications/tools/list_changed": + manifest_state["stale"] = True + + async with ClientSession(read, write, message_handler=message_handler) as session: yield session + @staticmethod + async def _list_all_tools(mcp_client: Any) -> tuple[list[Any], str | None]: + """Aggregate every tools/list page and fingerprint the complete snapshot.""" + capture = ToolManifestCapture() + tools: list[Any] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + while True: + page = await mcp_client.list_tools(cursor=cursor) if cursor is not None else await mcp_client.list_tools() + capture.observe_page(page, request_cursor=cursor) + page_tools, next_cursor = tools_page(page) + tools.extend(page_tools) + if next_cursor is None: + return tools, capture.fingerprint + if next_cursor in seen_cursors: + raise RuntimeError(f"tools/list pagination repeated cursor {next_cursor!r}") + seen_cursors.add(next_cursor) + cursor = next_cursor + def run_task( self, prompt: str, @@ -199,6 +233,7 @@ def run_task( cwd: Path | None = None, evidence_sentinels: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, ) -> AgentRun: if not model: raise ValueError("the API driver requires a model ID") @@ -214,6 +249,7 @@ def run_task( cwd=cwd, evidence_sentinels=evidence_sentinels, evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, ) ) @@ -228,11 +264,13 @@ async def _run_task( cwd: Path | None, evidence_sentinels: dict[str, Any] | None, evidence_targets: dict[str, Any] | None, + evidence_aggregates: dict[str, Any] | None, ) -> AgentRun: backend = self._make_backend(model) evidence = normalize_evidence_sentinels(evidence_sentinels) targets = normalize_evidence_targets(evidence_targets) - evidence_active = bool(configured_evidence_labels(evidence, targets)) + aggregates = normalize_evidence_aggregates(evidence_aggregates) + evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) calls: list[dict[str, Any]] = [] pending_results: list[tuple[int, str]] = [] usage_per_iteration: list[Usage] = [] @@ -248,12 +286,11 @@ async def _run_task( provider_stop_reason: str | None = None params = self._server_params(mcp_env, cwd) - async with self._mcp_session(params) as mcp_client: + manifest_state = {"stale": False} + tool_manifest_fingerprint: str | None = None + async with self._mcp_session(params, manifest_state=manifest_state) as mcp_client: await mcp_client.initialize() - tools_result = await mcp_client.list_tools() - raw_tools = ( - tools_result.get("tools", []) if isinstance(tools_result, dict) else getattr(tools_result, "tools", []) - ) + raw_tools, tool_manifest_fingerprint = await self._list_all_tools(mcp_client) backend.start(system, prompt, [tool_spec_from_mcp(tool) for tool in raw_tools]) # Match the historical metric: model/tool loop only, after list_tools. @@ -321,11 +358,23 @@ async def _run_task( calls[idx]["is_error"] = result.is_error calls[idx]["duration_ms"] = duration_ms if evidence_active: - calls[idx]["observed_sentinels"] = observed_sentinel_labels( - result.text, - evidence, - request_args=calls[idx]["args"], - evidence_targets=targets, + calls[idx]["observed_sentinels"] = sorted( + set( + observed_sentinel_labels( + result.text, + evidence, + request_args=calls[idx]["args"], + evidence_targets=targets, + ) + ) + | set( + observed_aggregate_labels( + result.text, + aggregates, + request_args=calls[idx]["args"], + evidence_targets=targets, + ) + ) ) pending_results.append((idx, result.text)) if matched_ids != set(call_indices) or len(call_indices) != len(turn.tool_calls): @@ -374,6 +423,8 @@ async def _run_task( "cache_creation_input_tokens": total_cache_creation_input_tokens, "source": "iterations", } + if manifest_state["stale"]: + tool_manifest_fingerprint = None return AgentRun( calls=calls, final_text=final_text, @@ -388,6 +439,9 @@ async def _run_task( usage_per_iteration=usage_per_iteration, cum_input_tokens=total_input_tokens, result_pair_mismatch=result_pair_mismatch, + trace_integrity=not result_pair_mismatch, + trace_integrity_reason="result_pair_mismatch" if result_pair_mismatch else None, + tool_manifest_fingerprint=tool_manifest_fingerprint, token_count_failures=token_count_failures, result_tokens_estimated=result_tokens_estimated, evidence_trace_available=evidence_active, @@ -425,6 +479,16 @@ class CliOutputError(RuntimeError): """Signal that vendor output could not produce a valid ``AgentRun``.""" +class CliRunError(RuntimeError): + """CLI failure retaining typed sidecar observations for the result row.""" + + def __init__(self, message: str, sidecar: ProxySidecarResult | None = None) -> None: + super().__init__(message) + self.trace_integrity = sidecar.trace_integrity if sidecar is not None else True + self.trace_integrity_reason = sidecar.trace_integrity_reason if sidecar is not None else None + self.tool_manifest_fingerprint = sidecar.tool_manifest_fingerprint if sidecar is not None else None + + class CliDriver(ABC): """Template for CLI drivers that run one MCP-backed subprocess task.""" @@ -534,6 +598,7 @@ def run_task( cwd: Path | None = None, evidence_sentinels: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, ) -> AgentRun: """Run one CLI task using the shared configuration/proxy/timeout flow.""" task_cwd = (cwd or REPO_ROOT).resolve() @@ -541,7 +606,10 @@ def run_task( self.validate_run() temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None - with tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td: + with ( + tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td, + tempfile.TemporaryDirectory(prefix="plane-eval-evidence-") as evidence_td, + ): temp_dir = Path(td) sidecar = temp_dir / "proxy-sidecar.jsonl" child_env = { @@ -549,7 +617,8 @@ def run_task( } evidence = normalize_evidence_sentinels(evidence_sentinels) targets = normalize_evidence_targets(evidence_targets) - evidence_active = bool(configured_evidence_labels(evidence, targets)) + aggregates = normalize_evidence_aggregates(evidence_aggregates) + evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) real_command = ( list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] ) @@ -557,8 +626,8 @@ def run_task( if self.use_proxy: evidence_path = None if evidence_active: - evidence_path = temp_dir / "proxy-evidence.json" - write_evidence_config(evidence_path, evidence, targets) + evidence_path = Path(evidence_td) / "proxy-evidence.json" + write_evidence_config(evidence_path, evidence, targets, aggregates) server_command = proxy_wrap_server_command( real_command, sidecar_path=sidecar, @@ -594,13 +663,20 @@ def run_task( calls: list[dict[str, Any]] = [] client_calls: list[dict[str, Any]] = [] call_source = self.default_call_source + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None if self.use_proxy: - calls, client_calls, call_source = harvest_proxy_after_cli_timeout( + sidecar_result = harvest_proxy_after_cli_timeout( calls, client_calls, sidecar, notes, ) + calls, client_calls, call_source = sidecar_result + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint evidence_available = False if evidence_active and call_source == "proxy": _proxy_calls, status = load_proxy_sidecar(sidecar) @@ -624,6 +700,9 @@ def run_task( hit_max_turns=False, wall_time_s=round(wall, 3), evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, experimental=self.experimental, notes=notes, ) @@ -637,20 +716,28 @@ def run_task( notes=notes, ) except CliOutputError as exc: + sidecar_result = None if self.use_proxy: - apply_proxy_sidecar([], [], sidecar, notes) + sidecar_result = apply_proxy_sidecar([], [], sidecar, notes) detail = "; ".join(notes) - raise RuntimeError(f"{exc}: {detail}") from None + raise CliRunError(f"{exc}: {detail}", sidecar_result) from None + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None if self.use_proxy: - calls, client_calls, proxy_source = apply_proxy_sidecar( + sidecar_result = apply_proxy_sidecar( output.calls, output.client_tool_calls, sidecar, notes, ) + calls, client_calls, proxy_source = sidecar_result output.calls = calls output.client_tool_calls = client_calls + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint if proxy_source == "proxy": output.call_source = "proxy" @@ -680,6 +767,9 @@ def run_task( hit_max_turns=output.hit_max_turns, wall_time_s=round(wall, 3), evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, experimental=self.experimental, notes=notes, ) @@ -694,6 +784,7 @@ def run_task( "CliLaunch", "CliOutput", "CliOutputError", + "CliRunError", "McpSessionFactory", "tool_result_from_mcp", "tool_spec_from_mcp", diff --git a/evals/evidence.py b/evals/evidence.py index af3331c0..851c5b42 100644 --- a/evals/evidence.py +++ b/evals/evidence.py @@ -12,6 +12,7 @@ import json import os from collections.abc import Mapping, Sequence +from hashlib import sha256 from pathlib import Path from typing import Any @@ -53,11 +54,86 @@ def normalize_evidence_targets(value: Any) -> dict[str, tuple[str, ...]]: return normalize_evidence_sentinels(value) -def configured_evidence_labels(sentinels: Any, targets: Any) -> tuple[str, ...]: +def normalize_evidence_aggregates(value: Any) -> dict[str, tuple[dict[str, Any], ...]]: + """Validate the two narrow aggregate response shapes used by R2 and R6.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[dict[str, Any], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[dict[str, Any]] = [] + for raw_spec in raw_specs: + if not isinstance(raw_spec, Mapping): + continue + kind = raw_spec.get("kind") + if kind == "total_count": + try: + specs.append({"kind": kind, "value": int(raw_spec["value"])}) + except (KeyError, TypeError, ValueError): + continue + elif kind == "grouped_counts" and isinstance(raw_spec.get("values"), Mapping): + try: + values = {str(key): int(count) for key, count in raw_spec["values"].items()} + except (TypeError, ValueError): + continue + if values: + specs.append({"kind": kind, "values": values}) + if specs: + normalized[label] = tuple(specs) + return normalized + + +def configured_evidence_labels(sentinels: Any, targets: Any, aggregates: Any = None) -> tuple[str, ...]: """Return labels that have both response values and target entity IDs.""" values_by_label = normalize_evidence_sentinels(sentinels) targets_by_label = normalize_evidence_targets(targets) - return tuple(sorted(values_by_label.keys() & targets_by_label.keys())) + aggregate_labels = normalize_evidence_aggregates(aggregates) + return tuple(sorted((values_by_label.keys() | aggregate_labels.keys()) & targets_by_label.keys())) + + +def fingerprint_evidence_sentinels(value: Any) -> dict[str, tuple[tuple[int, str], ...]]: + """Replace raw values with character lengths and one-way SHA-256 fingerprints.""" + return { + label: tuple((len(item), sha256(item.encode("utf-8")).hexdigest()) for item in values) + for label, values in normalize_evidence_sentinels(value).items() + } + + +def normalize_evidence_fingerprints(value: Any) -> dict[str, tuple[tuple[int, str], ...]]: + """Validate serialized response-value fingerprints, dropping malformed entries.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[tuple[int, str], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[tuple[int, str]] = [] + for raw_spec in raw_specs: + if isinstance(raw_spec, Mapping): + raw_length = raw_spec.get("length") + raw_digest = raw_spec.get("sha256") + elif ( + isinstance(raw_spec, Sequence) + and not isinstance(raw_spec, (str, bytes, bytearray)) + and len(raw_spec) == 2 + ): + raw_length, raw_digest = raw_spec + else: + continue + try: + length = int(raw_length) + except (TypeError, ValueError): + continue + digest = str(raw_digest or "").strip().lower() + if length > 0 and len(digest) == 64 and all(char in "0123456789abcdef" for char in digest): + specs.append((length, digest)) + clean = tuple(dict.fromkeys(specs)) + if clean: + normalized[label] = clean + return normalized def encode_evidence_sentinels(value: Any) -> str: @@ -77,50 +153,68 @@ def decode_evidence_sentinels(value: str | None) -> dict[str, tuple[str, ...]]: return normalize_evidence_sentinels(raw) -def encode_evidence_config(sentinels: Any, targets: Any) -> str: - """Serialize the proxy-only matching configuration.""" +def encode_evidence_config(sentinels: Any, targets: Any, aggregates: Any = None) -> str: + """Serialize targets plus one-way value fingerprints, never raw sentinels.""" + fingerprints = fingerprint_evidence_sentinels(sentinels) return json.dumps( { - "sentinels": normalize_evidence_sentinels(sentinels), + "fingerprints": { + label: [{"length": length, "sha256": digest} for length, digest in specs] + for label, specs in fingerprints.items() + }, "targets": normalize_evidence_targets(targets), + "aggregates": normalize_evidence_aggregates(aggregates), }, ensure_ascii=True, separators=(",", ":"), ) -def decode_evidence_config(value: str | None) -> tuple[dict[str, tuple[str, ...]], dict[str, tuple[str, ...]]]: +def decode_evidence_config( + value: str | None, +) -> tuple[ + dict[str, tuple[tuple[int, str], ...]], + dict[str, tuple[str, ...]], + dict[str, tuple[dict[str, Any], ...]], +]: """Decode proxy-only matching configuration, failing closed on malformed input.""" if not value: - return {}, {} + return {}, {}, {} try: raw = json.loads(value) except (TypeError, ValueError): - return {}, {} + return {}, {}, {} if not isinstance(raw, Mapping): - return {}, {} + return {}, {}, {} return ( - normalize_evidence_sentinels(raw.get("sentinels")), + normalize_evidence_fingerprints(raw.get("fingerprints")), normalize_evidence_targets(raw.get("targets")), + normalize_evidence_aggregates(raw.get("aggregates")), ) -def write_evidence_config(path: Path, sentinels: Any, targets: Any) -> None: +def write_evidence_config(path: Path, sentinels: Any, targets: Any, aggregates: Any = None) -> None: """Create a private, one-shot proxy configuration outside the agent cwd.""" - payload = encode_evidence_config(sentinels, targets) + payload = encode_evidence_config(sentinels, targets, aggregates) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as stream: stream.write(payload) -def consume_evidence_config(path: Path | None) -> tuple[dict[str, tuple[str, ...]], dict[str, tuple[str, ...]]]: +def consume_evidence_config( + path: Path | None, +) -> tuple[ + dict[str, tuple[tuple[int, str], ...]], + dict[str, tuple[str, ...]], + dict[str, tuple[dict[str, Any], ...]], +]: """Read and unlink a one-shot proxy configuration, failing closed.""" if path is None: - return {}, {} + return {}, {}, {} try: raw = path.read_text(encoding="utf-8") except OSError: - return {}, {} + return {}, {}, {} finally: try: path.unlink() @@ -137,7 +231,10 @@ def contains(value: Any) -> bool: return any(contains(item) for item in value.values()) if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): return any(contains(item) for item in value) - return value is not None and str(value) in targets + if value is None: + return False + text = str(value) + return any(target == text or target in text for target in targets) return contains(request_args) @@ -164,6 +261,112 @@ def observed_sentinel_labels( ) +def observed_fingerprint_labels( + response_text: str, + fingerprints: Any, + *, + request_args: Any, + evidence_targets: Any, +) -> list[str]: + """Match target-bound value fingerprints without ever receiving the raw values.""" + text = str(response_text or "") + if not text: + return [] + normalized = normalize_evidence_fingerprints(fingerprints) + targets = normalize_evidence_targets(evidence_targets) + eligible = { + label: specs + for label, specs in normalized.items() + if label in targets and _request_targets(request_args, targets[label]) + } + if not eligible: + return [] + + expected_by_length: dict[int, set[str]] = {} + labels_by_spec: dict[tuple[int, str], set[str]] = {} + for label, specs in eligible.items(): + for length, digest in specs: + expected_by_length.setdefault(length, set()).add(digest) + labels_by_spec.setdefault((length, digest), set()).add(label) + + matched: set[str] = set() + for length, expected in expected_by_length.items(): + if length > len(text): + continue + remaining = set(expected) + for start in range(len(text) - length + 1): + digest = sha256(text[start : start + length].encode("utf-8")).hexdigest() + if digest not in remaining: + continue + matched.update(labels_by_spec[(length, digest)]) + remaining.remove(digest) + if not remaining: + break + return sorted(matched) + + +def _decoded_documents(response_text: str) -> list[Any]: + """Decode JSON-RPC/MCP wrappers and JSON strings embedded inside them.""" + documents: list[Any] = [] + pending: list[Any] = [response_text] + seen_strings: set[str] = set() + while pending: + value = pending.pop() + documents.append(value) + if isinstance(value, Mapping): + pending.extend(value.values()) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + pending.extend(value) + elif isinstance(value, str): + text = value.strip() + if text in seen_strings or not text or text[0] not in "[{": + continue + seen_strings.add(text) + try: + pending.append(json.loads(text)) + except (TypeError, ValueError): + continue + return documents + + +def observed_aggregate_labels( + response_text: str, + aggregates: Any, + *, + request_args: Any, + evidence_targets: Any, +) -> list[str]: + """Match exact aggregate result fields while retaining seeded target binding.""" + specs_by_label = normalize_evidence_aggregates(aggregates) + targets_by_label = normalize_evidence_targets(evidence_targets) + documents = _decoded_documents(response_text) + matched: list[str] = [] + for label, specs in specs_by_label.items(): + targets = targets_by_label.get(label, ()) + for spec in specs: + if spec["kind"] == "total_count": + if not _request_targets(request_args, targets): + continue + if any(isinstance(value, Mapping) and value.get("total_count") == spec["value"] for value in documents): + matched.append(label) + break + elif spec["kind"] == "grouped_counts": + expected = spec["values"] + for value in documents: + if not isinstance(value, Mapping) or not isinstance(value.get("grouped_counts"), Mapping): + continue + grouped = value["grouped_counts"] + if all( + isinstance(grouped.get(target), Mapping) and grouped[target].get("count") == expected_count + for target, expected_count in expected.items() + ): + matched.append(label) + break + if label in matched: + break + return sorted(set(matched)) + + def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, target_ids: Sequence[Any]) -> None: """Register API-confirmed values and the entity IDs whose reads may prove them.""" clean_values: list[str] = [] @@ -176,13 +379,37 @@ def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, targe clean = tuple(dict.fromkeys(clean_values)) if not clean: raise RuntimeError("target evidence has no API-confirmed sentinel values") - clean_targets = tuple(dict.fromkeys(str(value).strip() for value in target_ids if str(value).strip())) + clean_targets = tuple( + dict.fromkeys(str(value).strip() for value in target_ids if value is not None and str(value).strip()) + ) if not clean_targets: raise RuntimeError("target evidence has no seeded target entity ids") context["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: clean} context["evidence_targets"] = {TARGET_ENTITY_EVIDENCE: clean_targets} +def set_target_count_evidence(context: dict[str, Any], count: int, *, target_ids: Sequence[Any]) -> None: + """Allow an exact ``total_count`` response whose request names the seeded target.""" + clean_targets = tuple(str(value).strip() for value in target_ids if value is not None and str(value).strip()) + if not clean_targets: + raise RuntimeError("target count evidence has no seeded target entity ids") + context.setdefault("evidence_targets", {})[TARGET_ENTITY_EVIDENCE] = clean_targets + context.setdefault("evidence_aggregates", {})[TARGET_ENTITY_EVIDENCE] = ( + {"kind": "total_count", "value": int(count)}, + ) + + +def set_target_grouped_count_evidence(context: dict[str, Any], values: Mapping[Any, int]) -> None: + """Allow grouped counts only when every seeded target id has its exact count.""" + clean = {str(target): int(count) for target, count in values.items() if str(target).strip()} + if not clean: + raise RuntimeError("target grouped-count evidence has no seeded targets") + context.setdefault("evidence_targets", {})[TARGET_ENTITY_EVIDENCE] = tuple(clean) + context.setdefault("evidence_aggregates", {})[TARGET_ENTITY_EVIDENCE] = ( + {"kind": "grouped_counts", "values": clean}, + ) + + __all__ = [ "EVIDENCE_SENTINELS_ENV", "TARGET_ENTITY_EVIDENCE", @@ -192,9 +419,16 @@ def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, targe "decode_evidence_sentinels", "encode_evidence_config", "encode_evidence_sentinels", + "fingerprint_evidence_sentinels", + "normalize_evidence_fingerprints", + "normalize_evidence_aggregates", "normalize_evidence_sentinels", "normalize_evidence_targets", "observed_sentinel_labels", + "observed_fingerprint_labels", + "observed_aggregate_labels", + "set_target_count_evidence", "set_target_evidence", + "set_target_grouped_count_evidence", "write_evidence_config", ] From 8177441d586b3ffb8c46b919458c1a00c3c1d98c Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 20:56:42 +0530 Subject: [PATCH 38/93] Count what the proxy lost, and identify the surface it recorded unmatched_responses counted normal protocol traffic, not loss: only tools/call was registered as pending, so every initialize and tools/list response incremented it. A clean session reported 2 with nothing missing, which would have branded every trace incomplete had it been made fatal. Every id-bearing request is now tracked, and the counter means what its name says. Unparsed lines, recorder-callback failures, sequence gaps against a persisted last_seq, and a lost non-tool response are all real loss and all fatal. The harness could not identify its own independent variable: server local|external plus a user-chosen label cannot show the tool surface was held fixed. tool_manifest_fingerprint hashes the full advertised descriptors across paginated pages, so two surfaces sharing a tool name but differing in schema no longer look identical. Co-Authored-By: Claude Opus 5 (1M context) --- evals/proxy.py | 159 ++++++++++-- evals/tool_manifest.py | 113 ++++++++ tests/evals/test_proxy.py | 419 +++++++++++++++++++++++++++++- tests/evals/test_tool_manifest.py | 71 +++++ 4 files changed, 734 insertions(+), 28 deletions(-) create mode 100644 evals/tool_manifest.py create mode 100644 tests/evals/test_tool_manifest.py diff --git a/evals/proxy.py b/evals/proxy.py index bb40e52a..5e025760 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -21,12 +21,16 @@ from evals.evidence import ( EVIDENCE_SENTINELS_ENV, - configured_evidence_labels, consume_evidence_config, + fingerprint_evidence_sentinels, + normalize_evidence_aggregates, + normalize_evidence_fingerprints, normalize_evidence_sentinels, normalize_evidence_targets, - observed_sentinel_labels, + observed_aggregate_labels, + observed_fingerprint_labels, ) +from evals.tool_manifest import ToolManifestCapture # Single post-EOF / child-exit deadline for the whole shutdown sequence. SHUTDOWN_DEADLINE_S = 10.0 @@ -137,19 +141,34 @@ def __init__( *, record_result_payloads: bool = False, evidence_sentinels: dict[str, Any] | None = None, + evidence_fingerprints: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, ) -> None: self.log_path = log_path self.record_result_payloads = record_result_payloads - self.evidence_sentinels = normalize_evidence_sentinels(evidence_sentinels) + raw_sentinels = normalize_evidence_sentinels(evidence_sentinels) + self.evidence_fingerprints = normalize_evidence_fingerprints(evidence_fingerprints) + if not self.evidence_fingerprints and raw_sentinels: + self.evidence_fingerprints = fingerprint_evidence_sentinels(raw_sentinels) self.evidence_targets = normalize_evidence_targets(evidence_targets) - self.evidence_active = bool(configured_evidence_labels(self.evidence_sentinels, self.evidence_targets)) + self.evidence_aggregates = normalize_evidence_aggregates(evidence_aggregates) + self.evidence_active = bool( + (self.evidence_fingerprints.keys() | self.evidence_aggregates.keys()) & self.evidence_targets.keys() + ) self._lock = threading.Lock() + self._error_lock = threading.Lock() self._pending: dict[Any, dict[str, Any]] = {} + self._non_tool_pending: dict[Any, dict[str, Any]] = {} + self._tool_manifest = ToolManifestCapture() self._seq = 0 self.relayed_lines = 0 self.unparsed_lines = 0 + self.non_json_lines = 0 + self.malformed_jsonrpc = 0 + self.recorder_errors = 0 self.unmatched_responses = 0 + self.non_tool_responses = 0 self.notifications = 0 self.server_requests = 0 self.child_killed = False @@ -173,11 +192,20 @@ def note_relayed(self) -> None: with self._lock: self.relayed_lines += 1 - def note_unparsed(self) -> None: + def note_unparsed(self, *, malformed_jsonrpc: bool = False) -> None: with self._lock: self.unparsed_lines += 1 + if malformed_jsonrpc: + self.malformed_jsonrpc += 1 + else: + self.non_json_lines += 1 self.relayed_lines += 1 + def note_recorder_error(self) -> None: + """Count a swallowed callback failure without depending on recorder state.""" + with self._error_lock: + self.recorder_errors += 1 + def on_client_message(self, obj: dict[str, Any]) -> None: """Handle a parsed JSON-RPC message from the client (parent → child).""" has_method = "method" in obj @@ -189,7 +217,15 @@ def on_client_message(self, obj: dict[str, Any]) -> None: if not has_method: return method = obj.get("method") + req_id = obj.get("id") if method != "tools/call": + params = obj.get("params") + cursor = params.get("cursor") if isinstance(params, dict) else None + with self._lock: + self._non_tool_pending[req_id] = { + "method": str(method), + "cursor": str(cursor) if cursor is not None else None, + } return params = obj.get("params") or {} if not isinstance(params, dict): @@ -198,7 +234,6 @@ def on_client_message(self, obj: dict[str, Any]) -> None: arguments = params.get("arguments") if arguments is None: arguments = {} - req_id = obj.get("id") with self._lock: self._seq += 1 self._pending[req_id] = { @@ -216,6 +251,8 @@ def on_server_message(self, obj: dict[str, Any]) -> None: if has_method and not has_id: with self._lock: self.notifications += 1 + if obj.get("method") == "notifications/tools/list_changed": + self._tool_manifest.invalidate() return if has_method and has_id: with self._lock: @@ -227,6 +264,16 @@ def on_server_message(self, obj: dict[str, Any]) -> None: req_id = obj.get("id") with self._lock: pending = self._pending.pop(req_id, None) + non_tool_pending = self._non_tool_pending.pop(req_id, None) if pending is None else None + if non_tool_pending is not None: + with self._lock: + self.non_tool_responses += 1 + if non_tool_pending["method"] == "tools/list" and isinstance(obj.get("result"), dict): + self._tool_manifest.observe_page( + obj["result"], + request_cursor=non_tool_pending["cursor"], + ) + return if pending is None: with self._lock: self.unmatched_responses += 1 @@ -255,11 +302,23 @@ def on_server_message(self, obj: dict[str, Any]) -> None: } if self.evidence_active: # Persist labels only. The matching values and result body stay in memory. - row["observed_sentinels"] = observed_sentinel_labels( - result_text, - self.evidence_sentinels, - request_args=pending["args"], - evidence_targets=self.evidence_targets, + row["observed_sentinels"] = sorted( + set( + observed_fingerprint_labels( + result_text, + self.evidence_fingerprints, + request_args=pending["args"], + evidence_targets=self.evidence_targets, + ) + ) + | set( + observed_aggregate_labels( + result_text, + self.evidence_aggregates, + request_args=pending["args"], + evidence_targets=self.evidence_targets, + ) + ) ) if self.record_result_payloads: row["result_text"] = result_text @@ -271,17 +330,27 @@ def write_meta(self) -> None: if self.finalized: self.post_finalize_appends += 1 return + with self._error_lock: + recorder_errors = self.recorder_errors row = { "row_type": "proxy_meta", "relayed_lines": self.relayed_lines, "unparsed_lines": self.unparsed_lines, + "non_json_lines": self.non_json_lines, + "malformed_jsonrpc": self.malformed_jsonrpc, + "recorder_errors": recorder_errors, "unmatched_responses": self.unmatched_responses, + "non_tool_responses": self.non_tool_responses, "notifications": self.notifications, "server_requests": self.server_requests, "pending_left": len(self._pending), + "non_tool_pending_left": len(self._non_tool_pending), + "last_seq": self._seq, + "tool_request_count": self._seq, "child_killed": self.child_killed, "pumps_alive": self.pumps_alive, "evidence_trace_available": self.evidence_active, + "tool_manifest_fingerprint": self._tool_manifest.fingerprint, } line = json.dumps(row, default=str, ensure_ascii=False) + "\n" with self.log_path.open("a", encoding="utf-8") as fh: @@ -289,8 +358,34 @@ def write_meta(self) -> None: self.finalized = True +def _valid_jsonrpc_object(obj: dict[str, Any]) -> bool: + """Validate the JSON-RPC 2.0 message envelope used by MCP stdio.""" + if obj.get("jsonrpc") != "2.0": + return False + has_method = "method" in obj + has_id = "id" in obj + if has_id and (isinstance(obj.get("id"), bool) or not isinstance(obj.get("id"), (str, int, float, type(None)))): + return False + if has_method: + if not isinstance(obj.get("method"), str) or "result" in obj or "error" in obj: + return False + params = obj.get("params") + return params is None or isinstance(params, (dict, list)) + if not has_id or ("result" in obj) == ("error" in obj): + return False + if "error" not in obj: + return True + error = obj.get("error") + return bool( + isinstance(error, dict) + and isinstance(error.get("code"), int) + and not isinstance(error.get("code"), bool) + and isinstance(error.get("message"), str) + ) + + def try_parse_json_line(line: bytes) -> dict[str, Any] | None: - """Parse a JSON object line; return None on failure (never raises).""" + """Parse a valid JSON-RPC object line; return None on failure (never raises).""" try: text = line.decode("utf-8").strip() except UnicodeDecodeError: @@ -301,7 +396,24 @@ def try_parse_json_line(line: bytes) -> dict[str, Any] | None: obj = json.loads(text) except json.JSONDecodeError: return None - return obj if isinstance(obj, dict) else None + return obj if isinstance(obj, dict) and _valid_jsonrpc_object(obj) else None + + +def classify_jsonrpc_line(line: bytes) -> tuple[str, dict[str, Any] | None]: + """Classify a framed line as blank, non-JSON, malformed JSON-RPC, or valid.""" + try: + text = line.decode("utf-8").strip() + except UnicodeDecodeError: + return "non_json", None + if not text: + return "blank", None + try: + obj = json.loads(text) + except json.JSONDecodeError: + return "non_json", None + if not isinstance(obj, dict) or not _valid_jsonrpc_object(obj): + return "malformed_jsonrpc", None + return "valid", obj def process_buffer_lines( @@ -327,9 +439,11 @@ def process_buffer_lines( line = bytes(buf[: idx + 1]) del buf[: idx + 1] if record_jsonrpc and recorder is not None: - obj = try_parse_json_line(line) - if obj is None: - recorder.note_unparsed() + classification, obj = classify_jsonrpc_line(line) + if classification == "blank": + recorder.note_relayed() + elif obj is None: + recorder.note_unparsed(malformed_jsonrpc=classification == "malformed_jsonrpc") else: recorder.note_relayed() try: @@ -338,7 +452,7 @@ def process_buffer_lines( else: recorder.on_server_message(obj) except Exception: - pass + recorder.note_recorder_error() # Forward only after recording so the opposite endpoint cannot race. write_all_fd(forward_fd, line) @@ -404,7 +518,7 @@ def pump_raw( except (BrokenPipeError, OSError): pass if record_jsonrpc and recorder is not None: - recorder.note_unparsed() + recorder.note_unparsed(malformed_jsonrpc=True) buf.clear() finally: done.set() @@ -432,7 +546,9 @@ def run_proxy( *, record_result_payloads: bool = False, evidence_sentinels: dict[str, Any] | None = None, + evidence_fingerprints: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, ) -> int: """Spawn ``command`` as the real MCP server and relay with recording. @@ -445,7 +561,9 @@ def run_proxy( log_path, record_result_payloads=record_result_payloads, evidence_sentinels=evidence_sentinels, + evidence_fingerprints=evidence_fingerprints, evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, ) child: subprocess.Popen[bytes] | None = None # Scrub repo PYTHONPATH so the real server does not import from this tree. @@ -630,13 +748,14 @@ def main(argv: list[str] | None = None) -> int: # Already a session leader, or platform forbids setsid — continue. pass args = parse_args(argv) - evidence_sentinels, evidence_targets = consume_evidence_config(args.evidence_file) + evidence_fingerprints, evidence_targets, evidence_aggregates = consume_evidence_config(args.evidence_file) return run_proxy( list(args.command), Path(args.log), record_result_payloads=bool(args.record_result_payloads), - evidence_sentinels=evidence_sentinels, + evidence_fingerprints=evidence_fingerprints, evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, ) diff --git a/evals/tool_manifest.py b/evals/tool_manifest.py new file mode 100644 index 00000000..5beaec80 --- /dev/null +++ b/evals/tool_manifest.py @@ -0,0 +1,113 @@ +"""Canonical, route-agnostic fingerprints for advertised MCP tool manifests.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + + +def _json_value(value: Any) -> Any: + """Return a stable JSON-compatible representation, omitting null object fields.""" + dump = getattr(value, "model_dump", None) + if callable(dump): + value = dump(by_alias=True, exclude_none=True) + elif not isinstance(value, (dict, list, tuple, str, int, float, bool)) and value is not None: + try: + value = vars(value) + except TypeError: + value = str(value) + + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items() if item is not None} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def canonical_tool_descriptors(tools: list[Any]) -> list[dict[str, Any]]: + """Canonicalize complete advertised descriptors and sort them by tool name.""" + descriptors: list[dict[str, Any]] = [] + for tool in tools: + descriptor = _json_value(tool) + if not isinstance(descriptor, dict): + descriptor = {"name": str(getattr(tool, "name", "")), "descriptor": descriptor} + descriptors.append(descriptor) + return sorted( + descriptors, + key=lambda item: ( + str(item.get("name") or ""), + json.dumps(item, sort_keys=True, separators=(",", ":"), ensure_ascii=False), + ), + ) + + +def tool_manifest_fingerprint(tools: list[Any]) -> str: + """Hash the complete canonical advertised tool descriptors.""" + payload = json.dumps( + canonical_tool_descriptors(tools), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def tools_page(page: Any) -> tuple[list[Any], str | None]: + """Extract the advertised tools and next cursor from a tools/list result page.""" + if isinstance(page, dict): + raw_tools = page.get("tools") + cursor = page.get("nextCursor", page.get("next_cursor")) + else: + raw_tools = getattr(page, "tools", None) + cursor = getattr(page, "nextCursor", None) + if cursor is None: + cursor = getattr(page, "next_cursor", None) + tools = list(raw_tools) if isinstance(raw_tools, (list, tuple)) else [] + return tools, str(cursor) if cursor is not None else None + + +@dataclass(slots=True) +class ToolManifestCapture: + """Aggregate one passive paginated tools/list snapshot.""" + + descriptors: list[Any] = field(default_factory=list) + expected_cursor: str | None = None + active: bool = False + fingerprint: str | None = None + + def observe_page(self, page: Any, *, request_cursor: str | None) -> None: + """Observe one response page; publish only a complete root-to-final snapshot.""" + if request_cursor is None: + self.descriptors = [] + self.expected_cursor = None + self.active = True + self.fingerprint = None + elif not self.active or request_cursor != self.expected_cursor: + self.invalidate() + return + + tools, next_cursor = tools_page(page) + self.descriptors.extend(tools) + self.expected_cursor = next_cursor + if next_cursor is None: + self.fingerprint = tool_manifest_fingerprint(self.descriptors) + self.active = False + + def invalidate(self) -> None: + """Drop a partial or stale snapshot.""" + self.descriptors = [] + self.expected_cursor = None + self.active = False + self.fingerprint = None + + +__all__ = [ + "ToolManifestCapture", + "canonical_tool_descriptors", + "tool_manifest_fingerprint", + "tools_page", +] diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 9c1f3da0..87a936ce 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import stat import subprocess import sys import textwrap @@ -16,7 +18,7 @@ load_proxy_sidecar, load_proxy_sidecar_calls, ) -from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE +from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE, write_evidence_config from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -27,10 +29,25 @@ write_all_fd, ) from evals.proxy import main as proxy_main +from evals.report.summary import summarize +from evals.results import AgentRun, TaskResult, agent_run_to_task_result +from evals.runner.live import _record_trace_infra from tests.evals.conftest import case_params REPO = Path(__file__).resolve().parents[2] + +def _request(request_id: int, method: str, params: dict | None = None) -> dict: + message = {"jsonrpc": "2.0", "id": request_id, "method": method} + if params is not None: + message["params"] = params + return message + + +def _response(request_id: int, result: object) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + FAKE_SERVER = textwrap.dedent( r""" import json, sys @@ -549,13 +566,29 @@ def _sidecar_recorder_unit(tmp_path): rec = SidecarRecorder( tmp_path / "a.jsonl", evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 8, + "method": "tools/call", + "params": {"name": "t", "arguments": {"work_item_id": "non-target"}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 8, + "result": {"content": [{"type": "text", "text": f"target={sentinel}"}], "isError": False}, + } ) rec.on_client_message( { "jsonrpc": "2.0", "id": 9, "method": "tools/call", - "params": {"name": "t", "arguments": {"a": 1}}, + "params": {"name": "t", "arguments": {"work_item_id": "target-1"}}, } ) rec.on_server_message( @@ -568,19 +601,58 @@ def _sidecar_recorder_unit(tmp_path): rec.write_meta() calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] - raw_call = next(row for row in raw_rows if row.get("row_type") != "proxy_meta") - assert len(calls) == 1 + raw_calls = [row for row in raw_rows if row.get("row_type") != "proxy_meta"] + assert len(calls) == 2 assert calls[0]["tool"] == "t" - assert calls[0]["args"] == {"a": 1} + assert calls[0]["args"] == {"work_item_id": "non-target"} + assert calls[1]["args"] == {"work_item_id": "target-1"} assert calls[0]["origin"] == "plane" assert "result_text" not in calls[0] - assert "result_text" not in raw_call - assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] - assert raw_call["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert all("result_text" not in row for row in raw_calls) + assert calls[0]["observed_sentinels"] == [] + assert calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert raw_calls[0]["observed_sentinels"] == [] + assert raw_calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert sentinel not in (tmp_path / "a.jsonl").read_text(encoding="utf-8") assert rec.finalized is True +def test_proxy_records_exact_target_bound_aggregate_evidence_without_payload(tmp_path): + path = tmp_path / "aggregate.jsonl" + rec = SidecarRecorder( + path, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1"]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [{"kind": "total_count", "value": 4}], + }, + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "count_work_items", + "arguments": {"pql": 'project = "project-1"'}, + }, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": '{"total_count": 4}'}]}, + } + ) + rec.write_meta() + + calls = load_proxy_sidecar_calls(path) + assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + persisted = path.read_text(encoding="utf-8") + assert "result_text" not in persisted + assert '"total_count"' not in persisted + + def _sidecar_result_payload_round_trips_only_when_enabled(tmp_path): path = tmp_path / "payload.jsonl" rec = SidecarRecorder(path, record_result_payloads=True) @@ -869,6 +941,42 @@ def test_scrub_child_pythonpath_removes_repo(): assert "PYTHONPATH" not in only +def test_proxy_consumes_private_evidence_file_before_starting_mcp(tmp_path: Path, monkeypatch): + sentinel = "hidden-target-fact-7b0a1f9c" + evidence_file = tmp_path / "evidence.json" + write_evidence_config( + evidence_file, + {TARGET_ENTITY_EVIDENCE: [sentinel]}, + {TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 + assert sentinel not in evidence_file.read_text(encoding="utf-8") + captured = {} + + def fake_run_proxy(command, log_path, **kwargs): + assert not evidence_file.exists(), "proxy must unlink evidence before starting Plane MCP" + captured.update({"command": command, "log_path": log_path, **kwargs}) + return 0 + + monkeypatch.setattr("evals.proxy.os.setsid", lambda: (_ for _ in ()).throw(OSError())) + monkeypatch.setattr("evals.proxy.run_proxy", fake_run_proxy) + rc = proxy_main( + [ + "--log", + str(tmp_path / "calls.jsonl"), + "--evidence-file", + str(evidence_file), + "--", + "fake-plane-mcp", + ] + ) + + assert rc == 0 + assert set(captured["evidence_fingerprints"]) == {TARGET_ENTITY_EVIDENCE} + assert captured["evidence_targets"] == {TARGET_ENTITY_EVIDENCE: ("target-1",)} + assert captured["evidence_aggregates"] == {} + + def test_rapid_response_pairing(tmp_path: Path): """Record-before-forward: fast child responses must pair with requests (no unmatched). @@ -1074,3 +1182,298 @@ def test_bounded_shutdown_wall_clock(tmp_path: Path): assert elapsed < SHUTDOWN_DEADLINE_S + 2.0 rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] assert rows[-1].get("row_type") == "proxy_meta" + + +def test_clean_protocol_session_is_complete_and_has_no_unmatched_responses(tmp_path: Path): + path = tmp_path / "clean.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "initialize")) + recorder.on_server_message(_response(1, {"protocolVersion": "2025-06-18"})) + recorder.on_client_message(_request(2, "tools/list")) + recorder.on_server_message(_response(2, {"tools": [{"name": "lookup", "inputSchema": {"type": "object"}}]})) + recorder.on_client_message(_request(3, "tools/call", {"name": "lookup", "arguments": {"query": "x"}})) + recorder.on_server_message(_response(3, {"content": [], "isError": False})) + recorder.write_meta() + + calls, status = load_proxy_sidecar(path) + notes: list[str] = [] + applied = apply_proxy_sidecar([], [], path, notes) + agent = agent_run_to_task_result( + AgentRun( + calls=applied.calls, + final_text="done", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + ) + ) + row = TaskResult(task_id="R1", expected_rows=1, success=True) + row.apply_agent_result(agent) + + assert status["state"] == "complete" + assert status["meta"]["unmatched_responses"] == 0 + assert status["meta"]["non_tool_responses"] == 2 + assert status["meta"]["non_tool_pending_left"] == 0 + assert status["tool_manifest_fingerprint"] + assert [call["tool"] for call in calls] == ["lookup"] + assert summarize([row], expected_rows=1).complete is True + + +def test_genuinely_lossy_trace_makes_summary_incomplete(tmp_path: Path, capsys): + path = tmp_path / "lossy.jsonl" + recorder = SidecarRecorder(path) + recorder.on_server_message(_response(404, {"content": []})) + recorder.write_meta() + notes: list[str] = [] + + applied = apply_proxy_sidecar([], [], path, notes) + agent = agent_run_to_task_result( + AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + notes=notes, + ) + ) + row = TaskResult(task_id="R1", expected_rows=1) + row.apply_agent_result(agent) + assert _record_trace_infra(row, agent, task={"id": "R1"}, repetition=0) is True + + summary = summarize([row], expected_rows=1) + assert row.error_class == "infra_trace" + assert summary.infra_errors == 1 + assert summary.complete is False + assert "unmatched_responses=1" in row.error + capsys.readouterr() + + +def test_lost_tools_list_response_is_non_tool_pending_loss(tmp_path: Path): + path = tmp_path / "lost-list.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "tools/list")) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["non_tool_pending_left"] == 1 + assert "non_tool_pending_left=1" in notes[0] + + +def test_deleting_final_call_row_is_caught_by_last_seq(tmp_path: Path): + path = tmp_path / "deleted-final.jsonl" + recorder = SidecarRecorder(path) + for request_id in (1, 2): + recorder.on_client_message(_request(request_id, "tools/call", {"name": f"tool-{request_id}", "arguments": {}})) + recorder.on_server_message(_response(request_id, {"content": []})) + recorder.write_meta() + rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + path.write_text( + "\n".join(json.dumps(row) for row in rows if row.get("seq") != 2) + "\n", + encoding="utf-8", + ) + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["missing_seq"] == 1 + assert "missing_seq=1" in notes[0] + + +@pytest.mark.parametrize( + ("sequences", "last_seq", "status_key"), + [ + pytest.param([0], 0, "invalid_seq", id="nonpositive"), + pytest.param([1, 1], 1, "duplicate_seq", id="duplicate"), + pytest.param([1, 3], 3, "missing_seq", id="gap"), + pytest.param([1, 2], 1, "unexpected_seq", id="past-last-seq"), + ], +) +def test_sidecar_rejects_invalid_duplicate_and_gapped_sequences( + tmp_path: Path, + sequences: list[int], + last_seq: int, + status_key: str, +): + path = tmp_path / f"{status_key}.jsonl" + rows = [{"tool": "t", "args": {}, "seq": seq} for seq in sequences] + rows.append( + { + "row_type": "proxy_meta", + "pending_left": 0, + "last_seq": last_seq, + "tool_request_count": last_seq, + } + ) + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status[status_key] > 0 + assert f"{status_key}={status[status_key]}" in notes[0] + + +@pytest.mark.parametrize("case", ["duplicate", "not-final"]) +def test_sidecar_requires_exactly_one_final_proxy_meta(tmp_path: Path, case: str): + path = tmp_path / f"meta-{case}.jsonl" + meta = {"row_type": "proxy_meta", "pending_left": 0, "last_seq": 0, "tool_request_count": 0} + if case == "duplicate": + rows = [meta, meta] + else: + meta.update({"last_seq": 1, "tool_request_count": 1}) + rows = [meta, {"tool": "late", "args": {}, "seq": 1}] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + if case == "duplicate": + assert status["proxy_meta_count"] == 2 + assert "proxy_meta_count=2" in notes[0] + else: + assert status["proxy_meta_not_final"] is True + assert "proxy_meta_not_final=1" in notes[0] + + +@pytest.mark.parametrize("case", ["missing-last-seq", "request-count-mismatch"]) +def test_sidecar_requires_consistent_sequence_metadata(tmp_path: Path, case: str): + path = tmp_path / f"sequence-meta-{case}.jsonl" + meta = {"row_type": "proxy_meta", "pending_left": 0, "tool_request_count": 0} + if case == "request-count-mismatch": + meta.update({"last_seq": 0, "tool_request_count": 1}) + path.write_text(json.dumps(meta) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["invalid_meta_fields"] > 0 + assert f"invalid_meta_fields={status['invalid_meta_fields']}" in notes[0] + + +def test_protocol_noise_and_malformed_jsonrpc_are_distinct_and_fatal(tmp_path: Path): + path = tmp_path / "protocol.jsonl" + recorder = SidecarRecorder(path) + read_fd, write_fd = os.pipe() + try: + buf = bytearray( + b' \nserver banner\n{"jsonrpc":"2.0"}\n{"jsonrpc":"2.0","method":"notifications/initialized"}\n' + ) + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=False, + record_jsonrpc=True, + ) + finally: + os.close(write_fd) + os.close(read_fd) + recorder.write_meta() + notes: list[str] = [] + applied = apply_proxy_sidecar([], [], path, notes) + + assert applied.trace_integrity is False + assert applied.trace_integrity_reason == "protocol_violation" + assert applied.status["unparsed_lines"] == 2 + assert applied.status["non_json_lines"] == 1 + assert applied.status["malformed_jsonrpc"] == 1 + assert "unparsed_lines=2" in notes[0] + agent = agent_run_to_task_result( + AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + notes=notes, + ) + ) + row = TaskResult(task_id="R1") + row.apply_agent_result(agent) + assert _record_trace_infra(row, agent, task={"id": "R1"}, repetition=0) is True + assert row.error_class == "infra_protocol" + + +def test_recorder_callback_failure_is_counted_and_invalidates_trace(tmp_path: Path, monkeypatch): + path = tmp_path / "recorder-error.jsonl" + recorder = SidecarRecorder(path) + + def fail(_obj): + raise RuntimeError("append failed") + + monkeypatch.setattr(recorder, "on_client_message", fail) + read_fd, write_fd = os.pipe() + try: + process_buffer_lines( + bytearray(b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n'), + forward_fd=write_fd, + recorder=recorder, + is_client=True, + record_jsonrpc=True, + ) + finally: + os.close(write_fd) + os.close(read_fd) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["recorder_errors"] == 1 + assert "recorder_errors=1" in notes[0] + + +def test_tools_list_changed_invalidates_proxy_manifest_snapshot(tmp_path: Path): + path = tmp_path / "stale-manifest.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "tools/list")) + recorder.on_server_message(_response(1, {"tools": [{"name": "lookup", "inputSchema": {"type": "object"}}]})) + recorder.on_server_message({"jsonrpc": "2.0", "method": "notifications/tools/list_changed"}) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["tool_manifest_fingerprint"] is None + + +def test_cli_fallback_does_not_restore_trace_integrity(tmp_path: Path): + path = tmp_path / "fallback.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(0, "tools/list")) + recorder.on_server_message(_response(0, {"tools": [{"name": "proxy-only", "inputSchema": {}}]})) + recorder.on_client_message(_request(1, "tools/call", {"name": "proxy-only", "arguments": {}})) + recorder.write_meta() + cli_calls = [ + {"tool": "cli-one", "args": {}, "origin": "plane"}, + {"tool": "cli-two", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + + applied = apply_proxy_sidecar(cli_calls, [], path, notes) + calls, _, source = applied + + assert source == "json" + assert calls == cli_calls + assert applied.trace_integrity is False + assert applied.trace_integrity_reason == "recorder_loss" + assert applied.tool_manifest_fingerprint is None + assert "proxy_sidecar_deferred_to_cli_trace" in notes diff --git a/tests/evals/test_tool_manifest.py b/tests/evals/test_tool_manifest.py new file mode 100644 index 00000000..ae33f2ff --- /dev/null +++ b/tests/evals/test_tool_manifest.py @@ -0,0 +1,71 @@ +"""Tool-manifest fingerprint regression tests.""" + +from __future__ import annotations + +from evals.tool_manifest import ToolManifestCapture, tool_manifest_fingerprint + + +def test_same_tool_names_with_different_schemas_have_different_manifest_fingerprints(): + short = [ + { + "name": "create_work_item", + "description": "Create an item", + "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}}}, + } + ] + consolidated = [ + { + "name": "create_work_item", + "description": "Create an item", + "inputSchema": { + "type": "object", + "properties": { + "workspace_slug": {"type": "string"}, + "project_slug": {"type": "string"}, + "title": {"type": "string"}, + "type_id": {"type": "string"}, + }, + }, + } + ] + + assert tool_manifest_fingerprint(short) != tool_manifest_fingerprint(consolidated) + + +def test_paginated_tools_list_hashes_like_equivalent_single_page(): + first = [{"name": "alpha", "inputSchema": {"type": "object"}}] + second = [{"name": "beta", "inputSchema": {"type": "object"}}] + paginated = ToolManifestCapture() + paginated.observe_page({"tools": first, "nextCursor": "page-2"}, request_cursor=None) + assert paginated.fingerprint is None + paginated.observe_page({"tools": second}, request_cursor="page-2") + + single = ToolManifestCapture() + single.observe_page({"tools": [*second, *first]}, request_cursor=None) + + assert paginated.fingerprint == single.fingerprint + + +def test_manifest_fingerprint_recursively_canonicalizes_object_key_order(): + left = [ + { + "name": "lookup", + "inputSchema": { + "type": "object", + "properties": {"q": {"type": "string", "description": "query"}}, + }, + "annotations": {"readOnlyHint": True, "destructiveHint": False}, + } + ] + right = [ + { + "annotations": {"destructiveHint": False, "readOnlyHint": True}, + "inputSchema": { + "properties": {"q": {"description": "query", "type": "string"}}, + "type": "object", + }, + "name": "lookup", + } + ] + + assert tool_manifest_fingerprint(left) == tool_manifest_fingerprint(right) From 0f7392fcafb366cfa96e9a81107aa911abade085 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 20:56:42 +0530 Subject: [PATCH 39/93] Refuse comparisons the persisted identity cannot establish compare.py contained no reference to battery, so two runs graded against different catalog revisions - different questions - were paired and printed with a Wilson interval and a permutation p-value. Battery mismatch is now an unconditional refusal across the summary, A/B and table paths, worded as comparability not being establishable rather than the comparison being inherently meaningless: the fingerprint also covers task selection. Other identity dimensions are treatments, not tolerated confounds, and are declared with --vary; battery cannot be waived. Identity is validated against raw rows before latest-wins dedupe, which can otherwise hide the conflicting row that proves a file is mixed. The headline aggregate now bootstraps task clusters. Pooled Wilson treats five repetitions of one task as five independent trials and reports an interval too narrow for exactly the number someone would quote. Co-Authored-By: Claude Opus 5 (1M context) --- evals/report/__init__.py | 26 +- evals/report/__main__.py | 1 + evals/report/command.py | 82 +++++- evals/report/compare.py | 38 ++- evals/report/identity.py | 221 +++++++++++++++ evals/report/load.py | 145 ++++++++-- evals/report/statistics.py | 16 ++ evals/report/summary.py | 52 +++- evals/report/table.py | 67 ++--- tests/evals/report/test_compare.py | 19 ++ tests/evals/report/test_identity.py | 402 ++++++++++++++++++++++++++++ tests/evals/report/test_summary.py | 51 +++- tests/evals/report/test_table.py | 29 +- 13 files changed, 1052 insertions(+), 97 deletions(-) create mode 100644 evals/report/identity.py create mode 100644 tests/evals/report/test_identity.py diff --git a/evals/report/__init__.py b/evals/report/__init__.py index 7f98faaf..2a22d4df 100644 --- a/evals/report/__init__.py +++ b/evals/report/__init__.py @@ -2,15 +2,28 @@ from .command import main from .compare import ab_compare, print_ab_report +from .identity import ( + ComparabilityError, + FileIdentity, + IdentityReport, + format_refusal, + identity_header_lines, + parse_varied_dimensions, + validate_persisted_identity, +) from .load import ( DedupeMode, ResultRow, + RunExpectation, + RunKeyValidation, dedupe_rows_latest, is_infra_error_row, is_meta_row, load_rows, + load_run_expectation, load_run_expected_rows, read_result, + validate_run_keys, ) from .statistics import iqr, median, paired_bootstrap_mean_ci, paired_permutation_pvalue, percentile, wilson_interval from .summary import ( @@ -36,11 +49,15 @@ result_tokens_marker, surface_label_for_file, task_sort_key, - warn_if_table_mixes_batteries, ) __all__ = [ "DedupeMode", + "RunExpectation", + "RunKeyValidation", + "ComparabilityError", + "FileIdentity", + "IdentityReport", "ResultRow", "ResultTokensMode", "Summary", @@ -52,6 +69,7 @@ "format_multi_rep_surface_cell", "format_number", "format_result_tokens", + "format_refusal", "format_surface_cell", "format_tool_distribution", "format_tool_variability", @@ -59,23 +77,27 @@ "iqr", "is_infra_error_row", "is_meta_row", + "identity_header_lines", "load_rows", + "load_run_expectation", "load_run_expected_rows", "main", "median", "paired_bootstrap_mean_ci", "paired_permutation_pvalue", + "parse_varied_dimensions", "percentile", "print_ab_report", "print_table", "prompt_excerpt", "read_result", + "validate_run_keys", "render_multi_surface_table", "result_tokens_marker", "result_tokens_mode", "summarize", "surface_label_for_file", "task_sort_key", - "warn_if_table_mixes_batteries", + "validate_persisted_identity", "wilson_interval", ] diff --git a/evals/report/__main__.py b/evals/report/__main__.py index 7c4483a6..615f8490 100644 --- a/evals/report/__main__.py +++ b/evals/report/__main__.py @@ -3,6 +3,7 @@ Usage: python -m evals.report evals/output/A.jsonl python -m evals.report A.jsonl B.jsonl # paired A/B bootstrap + permutation + python -m evals.report --vary resolved_model A.jsonl B.jsonl python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface python -m evals.report --table --markdown f1.jsonl f2.jsonl """ diff --git a/evals/report/command.py b/evals/report/command.py index 9a80f7e3..404d1a78 100644 --- a/evals/report/command.py +++ b/evals/report/command.py @@ -9,14 +9,27 @@ from evals.results import TaskResult from .compare import ab_compare, print_ab_report -from .load import DedupeMode, load_rows, load_run_expected_rows +from .identity import ( + ComparabilityError, + format_refusal, + identity_header_lines, + parse_varied_dimensions, + validate_persisted_identity, +) +from .load import ( + DedupeMode, + RunKeyValidation, + load_rows, + load_run_expectation, + load_run_expected_rows, + validate_run_keys, +) from .summary import summarize from .table import ( build_multi_surface_table, print_table, render_multi_surface_table, surface_label_for_file, - warn_if_table_mixes_batteries, ) @@ -37,12 +50,24 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="With --table, emit a GitHub-flavored markdown table", ) + parser.add_argument( + "--vary", + action="append", + default=[], + metavar="DIM[,DIM]", + help="Declare treatment dimensions (resolved_model, provider, driver, or server)", + ) parser.add_argument( "--no-dedupe", action="store_true", help="Keep all rows (forensics); default is latest-wins per (task_id,rep,label)", ) arguments = parser.parse_args(argv) + try: + varied_dimensions = parse_varied_dimensions(arguments.vary) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 dedupe: DedupeMode = "none" if arguments.no_dedupe else "latest" if not arguments.files: @@ -55,12 +80,43 @@ def main(argv: list[str] | None = None) -> int: print(f"error: file not found: {path}", file=sys.stderr) return 2 + if not arguments.table and len(paths) not in {1, 2}: + print( + "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", + file=sys.stderr, + ) + return 2 + + try: + identity = validate_persisted_identity(paths, varied_dimensions=varied_dimensions) + run_keys_by_path: dict[Path, RunKeyValidation | None] = {} + for path in paths: + expectation = load_run_expectation(path) + run_keys_by_path[path] = ( + validate_run_keys(load_rows(path, dedupe="none"), expectation) if expectation is not None else None + ) + except ComparabilityError as exc: + print(format_refusal(exc), file=sys.stderr) + return 2 + except ValueError as exc: + print(f"error: invalid run expectation: {exc}", file=sys.stderr) + return 2 + header_lines = identity_header_lines( + identity, + warn_missing_manifest=arguments.table or len(paths) == 2, + ) + for line in header_lines: + print(line) + if header_lines and arguments.table and arguments.markdown: + print() + if arguments.table: if len(paths) < 1: print("error: --table requires at least one JSONL", file=sys.stderr) return 2 labeled: list[tuple[str, list[TaskResult]]] = [] expected_by_label: dict[str, int | None] = {} + run_keys_by_label: dict[str, RunKeyValidation | None] = {} used_labels: set[str] = set() for path in paths: rows = load_rows(path, dedupe=dedupe) @@ -74,15 +130,23 @@ def main(argv: list[str] | None = None) -> int: used_labels.add(label) labeled.append((label, rows)) expected_by_label[label] = load_run_expected_rows(path) - warn_if_table_mixes_batteries(labeled) - table = build_multi_surface_table(labeled, expected_rows_by_column=expected_by_label) + run_keys_by_label[label] = run_keys_by_path[path] + table = build_multi_surface_table( + labeled, + expected_rows_by_column=expected_by_label, + run_keys_by_column=run_keys_by_label, + ) sys.stdout.write(render_multi_surface_table(table, markdown=arguments.markdown)) return 0 if all(values["complete"] for values in table["footer"].values()) else 1 if len(paths) == 1: path = paths[0] rows = load_rows(path, dedupe=dedupe) - summary = summarize(rows, expected_rows=load_run_expected_rows(path)) + summary = summarize( + rows, + expected_rows=load_run_expected_rows(path), + run_keys=run_keys_by_path[path], + ) print_table(summary, f"Summary: {path}") return 0 if summary.complete else 1 @@ -94,12 +158,10 @@ def main(argv: list[str] | None = None) -> int: rows_b, expected_rows_a=load_run_expected_rows(paths[0]), expected_rows_b=load_run_expected_rows(paths[1]), + run_keys_a=run_keys_by_path[paths[0]], + run_keys_b=run_keys_by_path[paths[1]], ) print_ab_report(comparison, paths[0], paths[1]) return 0 if comparison["summary_a"].complete and comparison["summary_b"].complete else 1 - print( - "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", - file=sys.stderr, - ) - return 2 + raise AssertionError("validated report arity did not select a command path") diff --git a/evals/report/compare.py b/evals/report/compare.py index a5a659ab..49d114dd 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue from .summary import completeness_statement, execution_coverage_statement, summarize from .table import format_number @@ -18,6 +18,8 @@ def ab_compare( *, expected_rows_a: int | None = None, expected_rows_b: int | None = None, + run_keys_a: RunKeyValidation | None = None, + run_keys_b: RunKeyValidation | None = None, ) -> dict[str, Any]: """Compare two result sets with task-paired call and success deltas. @@ -29,14 +31,14 @@ def ab_compare( labels, then bootstrap whole task pairs. This treats tasks as independent sampling units and assumes the labels cover comparable task instances. """ - summary_a = summarize(rows_a, expected_rows=expected_rows_a) - summary_b = summarize(rows_b, expected_rows=expected_rows_b) + summary_a = summarize(rows_a, expected_rows=expected_rows_a, run_keys=run_keys_a) + summary_b = summarize(rows_b, expected_rows=expected_rows_b, run_keys=run_keys_b) def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: output: dict[str, list[float]] = defaultdict(list) for raw_row in rows: row = read_result(raw_row) - if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped or not row.trace_integrity: continue if not row.success: continue @@ -96,6 +98,9 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: summary_a.aggregate_wilson_lo, summary_a.aggregate_wilson_hi, ), + "task_mean": summary_a.task_mean_success, + "task_cluster": (summary_a.task_cluster_lo, summary_a.task_cluster_hi), + "task_n": sum(task.n > 0 for task in summary_a.tasks.values()), }, "success_b": { "k": summary_b.aggregate_k, @@ -104,6 +109,9 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: summary_b.aggregate_wilson_lo, summary_b.aggregate_wilson_hi, ), + "task_mean": summary_b.task_mean_success, + "task_cluster": (summary_b.task_cluster_lo, summary_b.task_cluster_hi), + "task_n": sum(task.n > 0 for task in summary_b.tasks.values()), }, } @@ -113,14 +121,20 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N success_a, success_b = comparison["success_a"], comparison["success_b"] rate_a = (success_a["k"] / success_a["n"]) if success_a["n"] else 0.0 rate_b = (success_b["k"] / success_b["n"]) if success_b["n"] else 0.0 - print( - f" success A: {success_a['k']}/{success_a['n']} ({rate_a:.1%}) " - f"Wilson95 [{success_a['wilson'][0]:.2f},{success_a['wilson'][1]:.2f}]" - ) - print( - f" success B: {success_b['k']}/{success_b['n']} ({rate_b:.1%}) " - f"Wilson95 [{success_b['wilson'][0]:.2f},{success_b['wilson'][1]:.2f}]" - ) + for label, success, pooled_rate in (("A", success_a, rate_a), ("B", success_b, rate_b)): + task_mean = success["task_mean"] + task_lo, task_hi = success["task_cluster"] + if task_mean is None or task_lo is None or task_hi is None: + print(f" success {label} task-cluster: n/a (no evaluated tasks)") + else: + print( + f" success {label} task-cluster: {task_mean:.1%} " + f"cluster-bootstrap95 [{task_lo:.2f},{task_hi:.2f}] (n={success['task_n']} tasks)" + ) + print( + f" success {label} pooled repetitions: {success['k']}/{success['n']} ({pooled_rate:.1%}) " + f"Wilson95 [{success['wilson'][0]:.2f},{success['wilson'][1]:.2f}]" + ) print(f" A {execution_coverage_statement(comparison['summary_a'])}") print(f" B {execution_coverage_statement(comparison['summary_b'])}") print(f" A {completeness_statement(comparison['summary_a'])}") diff --git a/evals/report/identity.py b/evals/report/identity.py new file mode 100644 index 00000000..24d35c22 --- /dev/null +++ b/evals/report/identity.py @@ -0,0 +1,221 @@ +"""Persisted run-identity validation shared by every report command path.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +MISSING = "" +IDENTITY_FIELDS = ("battery", "resolved_model", "provider", "driver", "server") +VARYABLE_DIMENSIONS = ("resolved_model", "provider", "driver", "server") +TOOL_MANIFEST_FIELD = "tool_manifest_fingerprint" + + +class ComparabilityError(ValueError): + """The persisted records do not establish the requested comparison.""" + + def __init__(self, details: Iterable[str]) -> None: + self.details = tuple(details) + super().__init__("; ".join(self.details)) + + +@dataclass(frozen=True, slots=True) +class FileIdentity: + """Validated canonical identity and realized-model observations for one file.""" + + path: Path + values: dict[str, str] + realized_models: tuple[str, ...] + + @property + def realized_model_changed(self) -> bool: + return len(self.realized_models) > 1 + + +@dataclass(frozen=True, slots=True) +class IdentityReport: + """Identity evidence safe to print beside report measurements.""" + + files: tuple[FileIdentity, ...] + varied_dimensions: tuple[str, ...] + + +def persisted_value(value: Any) -> str: + """Normalize absent and empty persisted values to an explicit, non-wildcard value.""" + if value is None or value == "": + return MISSING + return str(value) + + +def parse_varied_dimensions(raw_values: Iterable[str]) -> tuple[str, ...]: + """Parse repeatable comma-separated --vary declarations.""" + requested: list[str] = [] + for raw_value in raw_values: + requested.extend(part.strip() for part in raw_value.split(",")) + if not requested: + return () + if any(not dimension for dimension in requested): + raise ValueError("--vary requires a dimension name") + if "all" in requested: + raise ValueError("--vary has no 'all'; name each treatment dimension individually") + if "battery" in requested: + raise ValueError("--vary battery is not allowed; the measurement universe cannot be waived") + unknown = sorted(set(requested) - set(VARYABLE_DIMENSIONS)) + if unknown: + valid = ", ".join(VARYABLE_DIMENSIONS) + raise ValueError(f"unknown --vary dimension(s): {', '.join(unknown)}; choose from: {valid}") + requested_set = set(requested) + return tuple(dimension for dimension in VARYABLE_DIMENSIONS if dimension in requested_set) + + +def _read_records(path: Path) -> tuple[list[tuple[int, dict[str, Any]]], list[tuple[int, dict[str, Any]]]]: + headers: list[tuple[int, dict[str, Any]]] = [] + rows: list[tuple[int, dict[str, Any]]] = [] + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + record = {} + if not isinstance(record, dict): + record = {} + target = headers if record.get("row_type") == "meta" else rows + target.append((line_number, record)) + return headers, rows + + +def _record_values(records: list[tuple[int, dict[str, Any]]], field: str) -> dict[str, list[int]]: + values: dict[str, list[int]] = {} + for line_number, record in records: + values.setdefault(persisted_value(record.get(field)), []).append(line_number) + return values + + +def _format_values(values: dict[str, list[int]]) -> str: + return ", ".join(f"{value} (line(s) {','.join(str(line) for line in lines)})" for value, lines in values.items()) + + +def _validate_file(path: Path) -> tuple[FileIdentity, list[str]]: + headers, rows = _read_records(path) + issues: list[str] = [] + values: dict[str, str] = {} + + for field in IDENTITY_FIELDS: + header_values = _record_values(headers, field) + row_values = _record_values(rows, field) + if len(header_values) > 1: + issues.append(f"{path}: conflicting meta headers for {field}: {_format_values(header_values)}") + if len(row_values) > 1: + issues.append(f"{path}: rows disagree on {field}: {_format_values(row_values)}") + if header_values and row_values and set(header_values) != set(row_values): + issues.append( + f"{path}: meta header disagrees with raw rows on {field}: " + f"header={_format_values(header_values)}; rows={_format_values(row_values)}" + ) + source = row_values or header_values or {MISSING: []} + values[field] = next(iter(source)) + + manifest_values = _record_values(rows, TOOL_MANIFEST_FIELD) + observed_manifests = {value: lines for value, lines in manifest_values.items() if value != MISSING} + if len(observed_manifests) > 1: + issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(observed_manifests)}") + values[TOOL_MANIFEST_FIELD] = next(iter(observed_manifests), MISSING) + + realized_models = tuple(sorted(_record_values(rows, "model"))) + return FileIdentity(path=path, values=values, realized_models=realized_models), issues + + +def validate_persisted_identity( + paths: Iterable[Path], + *, + varied_dimensions: Iterable[str] = (), +) -> IdentityReport: + """Validate files internally and against each other before any dedupe or statistics.""" + varied = tuple(varied_dimensions) + identities: list[FileIdentity] = [] + issues: list[str] = [] + for path in paths: + identity, file_issues = _validate_file(path) + identities.append(identity) + issues.extend(file_issues) + + if len(identities) > 1: + for field in IDENTITY_FIELDS: + by_path = {str(identity.path): identity.values[field] for identity in identities} + if len(set(by_path.values())) <= 1: + continue + if field != "battery" and field in varied: + continue + detail = "; ".join(f"{path}={value}" for path, value in by_path.items()) + issues.append(f"{field} differs across files: {detail}") + + if issues: + raise ComparabilityError(issues) + return IdentityReport(files=tuple(identities), varied_dimensions=varied) + + +def format_refusal(error: ComparabilityError) -> str: + """Render an exit-2 refusal without any report measurements.""" + lines = ["error: comparability cannot be established from the persisted identity"] + lines.extend(f" - {detail}" for detail in error.details) + return "\n".join(lines) + + +def identity_header_lines(report: IdentityReport, *, warn_missing_manifest: bool = False) -> list[str]: + """Render treatment declarations and non-canonical realized-model evidence.""" + lines: list[str] = [] + varied = report.varied_dimensions + if varied: + treatment = ", ".join(varied) + if len(varied) > 1: + treatment += " — end-to-end comparison; effect not attributable to any single dimension" + elif varied == ("driver",): + treatment += " — end-to-end driver question; cannot support a surface-only claim" + lines.append(f"Treatment: {treatment}") + if "driver" in varied and len(varied) > 1: + lines.append("Driver interpretation: end-to-end driver question; cannot support a surface-only claim") + for dimension in varied: + values = "; ".join(f"{identity.path}={identity.values[dimension]}" for identity in report.files) + lines.append(f"Treatment values ({dimension}): {values}") + + evidence = [ + identity for identity in report.files if identity.realized_models and identity.realized_models != (MISSING,) + ] + if evidence: + detail = "; ".join(f"{identity.path}={','.join(identity.realized_models)}" for identity in evidence) + lines.append(f"Realized model evidence: {detail}") + for identity in evidence: + if identity.realized_model_changed: + lines.append( + f"WARNING: realized model changed within {identity.path}: {','.join(identity.realized_models)}" + ) + if any(identity.values[TOOL_MANIFEST_FIELD] != MISSING for identity in report.files): + manifests = "; ".join(f"{identity.path}={identity.values[TOOL_MANIFEST_FIELD]}" for identity in report.files) + lines.append(f"Tool manifest evidence: {manifests}") + missing_manifests = [ + str(identity.path) for identity in report.files if identity.values[TOOL_MANIFEST_FIELD] == MISSING + ] + if warn_missing_manifest and missing_manifests: + lines.append("WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified for " + ", ".join(missing_manifests)) + return lines + + +__all__ = [ + "ComparabilityError", + "FileIdentity", + "IDENTITY_FIELDS", + "IdentityReport", + "MISSING", + "TOOL_MANIFEST_FIELD", + "VARYABLE_DIMENSIONS", + "format_refusal", + "identity_header_lines", + "parse_varied_dimensions", + "persisted_value", + "validate_persisted_identity", +] diff --git a/evals/report/load.py b/evals/report/load.py index 3742ccc0..4cab84f1 100644 --- a/evals/report/load.py +++ b/evals/report/load.py @@ -4,15 +4,135 @@ import json import sys +from collections import Counter, defaultdict +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal +from evals.result_lifecycle import is_terminal_result from evals.results import TaskResult DedupeMode = Literal["latest", "none"] ResultRow = TaskResult | dict[str, Any] +@dataclass(frozen=True, slots=True) +class RunExpectation: + """Exact task/repetition universe declared by a result-file meta header.""" + + task_ids: tuple[str, ...] + reps: int + label: str | None = None + + @property + def expected_rows(self) -> int: + return len(self.task_ids) * self.reps + + @property + def keys(self) -> frozenset[tuple[str, int]]: + return frozenset((task_id, rep) for task_id in self.task_ids for rep in range(self.reps)) + + +@dataclass(frozen=True, slots=True) +class RunKeyValidation: + """Raw-row comparison against one exact run expectation.""" + + expectation: RunExpectation + missing: tuple[str, ...] + unexpected: tuple[str, ...] + + @property + def exact(self) -> bool: + return not self.missing and not self.unexpected + + +def _first_meta_row(path: Path) -> dict[str, Any] | None: + with path.open(encoding="utf-8") as file: + for line in file: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(row, dict) or row.get("row_type") != "meta": + return None + return row + return None + + +def load_run_expectation(path: Path) -> RunExpectation | None: + """Reconstruct the exact expected ``(task_id, rep)`` set when declared.""" + row = _first_meta_row(path) + if row is None: + return None + raw_task_ids = row.get("expected_task_ids") + raw_reps = row.get("expected_reps") + if raw_task_ids is None and raw_reps is None: + return None + if not isinstance(raw_task_ids, list) or raw_reps is None: + raise ValueError(f"{path}: meta must declare expected_task_ids and expected_reps together") + task_ids = tuple(str(task_id) for task_id in raw_task_ids) + try: + reps = int(raw_reps) + except (TypeError, ValueError) as exc: + raise ValueError(f"{path}: expected_reps must be a positive integer") from exc + if not task_ids or any(not task_id for task_id in task_ids): + raise ValueError(f"{path}: expected_task_ids must be a non-empty list of non-empty ids") + if len(set(task_ids)) != len(task_ids): + raise ValueError(f"{path}: expected_task_ids contains duplicates") + if reps < 1: + raise ValueError(f"{path}: expected_reps must be a positive integer") + expectation = RunExpectation(task_ids=task_ids, reps=reps, label=str(row["label"]) if row.get("label") else None) + declared_rows = row.get("expected_rows") + if declared_rows is not None and int(declared_rows) != expectation.expected_rows: + raise ValueError( + f"{path}: expected_rows={declared_rows} disagrees with exact expectation={expectation.expected_rows}" + ) + return expectation + + +def _format_run_key(key: tuple[str, int], count: int) -> str: + rendered = f"{key[0]}[rep={key[1]}]" + return f"{rendered} x{count}" if count > 1 else rendered + + +def validate_run_keys(rows: list[ResultRow], expectation: RunExpectation) -> RunKeyValidation: + """Validate exact keys while allowing append-only retry history. + + For an expected key, every occurrence except the last must be retryable. The final + occurrence is authoritative. A prior terminal occurrence is a genuine duplicate and + remains visible as an unexpected key. + """ + expected = Counter({key: 1 for key in expectation.keys}) + histories: dict[tuple[str, int, str | None], list[TaskResult]] = defaultdict(list) + for raw_row in rows: + row = read_result(raw_row) + if not is_meta_row(row): + row_label = row.label if expectation.label is not None else None + histories[(row.task_id, row.rep, row_label)].append(row) + expected_history_keys = {(task_id, rep, expectation.label) for task_id, rep in expectation.keys} + observed = Counter( + {(task_id, rep): len(histories.get((task_id, rep, expectation.label), ())) for task_id, rep in expectation.keys} + ) + missing_counts = expected - observed + unexpected_counts: Counter[tuple[str, int]] = Counter() + for history_key, history in histories.items(): + key = history_key[:2] + if history_key not in expected_history_keys: + unexpected_counts[key] += len(history) + continue + terminal_predecessors = sum(is_terminal_result(row) for row in history[:-1]) + if terminal_predecessors: + unexpected_counts[key] += terminal_predecessors + return RunKeyValidation( + expectation=expectation, + missing=tuple(_format_run_key(key, count) for key, count in sorted(missing_counts.items())), + unexpected=tuple(_format_run_key(key, count) for key, count in sorted(unexpected_counts.items())), + ) + + def _invalid_result_row(path: Path, line_number: int, reason: str) -> TaskResult: """Represent an unreadable persisted row as a completeness-visible harness error.""" return TaskResult( @@ -27,22 +147,13 @@ def _invalid_result_row(path: Path, line_number: int, reason: str) -> TaskResult def load_run_expected_rows(path: Path) -> int | None: """Read the declared run size from the JSONL meta header, when available.""" - with path.open(encoding="utf-8") as file: - for line in file: - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - if not isinstance(row, dict): - continue - if row.get("row_type") != "meta": - return None - value = row.get("expected_rows") - return int(value) if value is not None else None - return None + if expectation := load_run_expectation(path): + return expectation.expected_rows + row = _first_meta_row(path) + if row is None: + return None + value = row.get("expected_rows") + return int(value) if value is not None else None def read_result(row: ResultRow) -> TaskResult: @@ -135,7 +246,7 @@ def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: if key in seen_keys: print( f"warning: {path}: duplicate (task_id, rep, label)={key} " - f"(--no-dedupe keeps all rows; bare --out reuse double-counts)", + "(--no-dedupe keeps append-only history; validator distinguishes retries from duplicates)", file=sys.stderr, ) else: diff --git a/evals/report/statistics.py b/evals/report/statistics.py index a91239af..d511c40a 100644 --- a/evals/report/statistics.py +++ b/evals/report/statistics.py @@ -113,6 +113,22 @@ def paired_bootstrap_mean_ci( return (percentile(bootstrap_means, tail), percentile(bootstrap_means, 1.0 - tail)) +def cluster_bootstrap_mean_ci( + task_rates: list[float], + *, + confidence: float = 0.95, + resamples: int = BOOTSTRAP_RESAMPLES, + seed: int = 0, +) -> tuple[float | None, float | None]: + """Bootstrap the mean success rate by resampling whole task clusters.""" + return paired_bootstrap_mean_ci( + task_rates, + confidence=confidence, + resamples=resamples, + seed=seed, + ) + + def median(values: list[float]) -> float | None: if not values: return None diff --git a/evals/report/summary.py b/evals/report/summary.py index 14c9888e..a0b19d9a 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -9,8 +9,8 @@ from evals.results import TaskResult from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family -from .load import ResultRow, is_infra_error_row, is_meta_row, read_result -from .statistics import iqr, median, percentile, wilson_interval +from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .statistics import cluster_bootstrap_mean_ci, iqr, median, percentile, wilson_interval ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] @@ -69,6 +69,7 @@ class Summary: expected_skips: int unexpected_skips: int cleanup_errors: int + trace_invalid_rows: int expected_skip_reasons: dict[str, int] unexpected_skip_reasons: dict[str, int] skipped_task_reasons: dict[str, list[str]] @@ -76,6 +77,11 @@ class Summary: aggregate_n: int aggregate_wilson_lo: float aggregate_wilson_hi: float + task_mean_success: float | None + task_cluster_lo: float | None + task_cluster_hi: float | None + missing_run_keys: tuple[str, ...] + unexpected_run_keys: tuple[str, ...] multi_rep: bool result_tokens_mode: ResultTokensMode @@ -83,10 +89,13 @@ class Summary: def complete(self) -> bool: return ( self.completed_rows == self.expected_rows + and not self.missing_run_keys + and not self.unexpected_run_keys and self.infra_errors == 0 and self.harness_errors == 0 and self.unexpected_skips == 0 and self.cleanup_errors == 0 + and self.trace_invalid_rows == 0 ) @property @@ -111,6 +120,8 @@ def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: labels: set[str] = set() for raw_row in rows: row = read_result(raw_row) + if not row.trace_integrity: + continue for call in row.calls: if call.result_tokens is None: continue @@ -153,6 +164,12 @@ def completeness_statement(summary: Summary) -> str: parts.append(f"unexpected skips={summary.unexpected_skips} [{reasons}]") if summary.cleanup_errors: parts.append(f"cleanup errors={summary.cleanup_errors}") + if summary.trace_invalid_rows: + parts.append(f"trace-invalid rows={summary.trace_invalid_rows}") + if summary.missing_run_keys: + parts.append(f"missing keys=[{', '.join(summary.missing_run_keys)}]") + if summary.unexpected_run_keys: + parts.append(f"unexpected keys=[{', '.join(summary.unexpected_run_keys)}]") if summary.expected_skips: reasons = _format_reason_counts(summary.expected_skip_reasons) parts.append(f"expected skips={summary.expected_skips} [{reasons}]") @@ -175,7 +192,12 @@ def execution_coverage_statement(summary: Summary) -> str: return "EXECUTION COVERAGE: " + "; ".join(parts) -def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Summary: +def summarize( + rows: list[ResultRow], + *, + expected_rows: int | None = None, + run_keys: RunKeyValidation | None = None, +) -> Summary: """Aggregate per-task metrics. Rows with ``error_class`` starting ``infra_`` are excluded from success-rate @@ -193,6 +215,7 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum expected_skips = 0 unexpected_skips = 0 cleanup_errors = 0 + trace_invalid_rows = 0 expected_skip_reasons: dict[str, int] = defaultdict(int) unexpected_skip_reasons: dict[str, int] = defaultdict(int) skipped_task_reasons: dict[str, set[str]] = defaultdict(set) @@ -204,6 +227,8 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum declared_expected_rows = max(declared_expected_rows, row.expected_rows) task_id = row.task_id repetitions_by_task[task_id].add(row.rep) + if not row.trace_integrity: + trace_invalid_rows += 1 if row.cleanup_error: cleanup_errors += 1 if is_infra_error_row(row): @@ -217,7 +242,7 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum if row.skipped: family = skip_reason_family(row.skipped) skipped_task_reasons[row.skipped].add(task_id) - if is_expected_environment_capability_skip(row.skipped): + if is_expected_environment_capability_skip(row.skipped, task_id=task_id): expected_skips += 1 expected_skip_reasons[family] += 1 completed_rows += 1 @@ -241,11 +266,11 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum total_passes += pass_count total_repetitions += repetition_count lower, upper = wilson_interval(pass_count, repetition_count) if repetition_count else (0.0, 0.0) - successful_calls = [float(row.num_calls) for row in task_results if row.success] + successful_calls = [float(row.num_calls) for row in task_results if row.success and row.trace_integrity] first_quartile, median_calls, third_quartile = iqr(successful_calls) minimum_calls = min(successful_calls) if successful_calls else None maximum_calls = max(successful_calls) if successful_calls else None - successful_results = [row for row in task_results if row.success] + successful_results = [row for row in task_results if row.success and row.trace_integrity] tool_reps = len(successful_results) failed_tool_reps = ( repetition_count @@ -270,6 +295,8 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum errored_calls = 0 result_tokens: list[float] = [] for row in task_results: + if not row.trace_integrity: + continue for call in row.calls: if call.is_error: errored_calls += 1 @@ -304,8 +331,13 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum aggregate_lower, aggregate_upper = ( wilson_interval(total_passes, total_repetitions) if total_repetitions else (0.0, 0.0) ) + task_rates = [task.k / task.n for task in output.values() if task.n] + task_mean_success = sum(task_rates) / len(task_rates) if task_rates else None + task_cluster_lower, task_cluster_upper = cluster_bootstrap_mean_ci(task_rates) resolved_expected_rows = ( - max(expected_rows, declared_expected_rows) + run_keys.expectation.expected_rows + if run_keys is not None + else max(expected_rows, declared_expected_rows) if expected_rows is not None else declared_expected_rows or sum(1 for row in rows if not is_meta_row(row)) ) @@ -319,6 +351,7 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum expected_skips=expected_skips, unexpected_skips=unexpected_skips, cleanup_errors=cleanup_errors, + trace_invalid_rows=trace_invalid_rows, expected_skip_reasons=dict(expected_skip_reasons), unexpected_skip_reasons=dict(unexpected_skip_reasons), skipped_task_reasons={reason: sorted(task_ids) for reason, task_ids in sorted(skipped_task_reasons.items())}, @@ -326,6 +359,11 @@ def summarize(rows: list[ResultRow], *, expected_rows: int | None = None) -> Sum aggregate_n=total_repetitions, aggregate_wilson_lo=aggregate_lower, aggregate_wilson_hi=aggregate_upper, + task_mean_success=task_mean_success, + task_cluster_lo=task_cluster_lower, + task_cluster_hi=task_cluster_upper, + missing_run_keys=run_keys.missing if run_keys is not None else (), + unexpected_run_keys=run_keys.unexpected if run_keys is not None else (), multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), ) diff --git a/evals/report/table.py b/evals/report/table.py index 9b602bb4..ddc86bea 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -2,7 +2,6 @@ from __future__ import annotations -import sys from collections import defaultdict from pathlib import Path from typing import Any @@ -10,7 +9,7 @@ from evals.results import TaskResult from evals.tasks import TASKS_BY_ID -from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .statistics import wilson_interval from .summary import ( Summary, @@ -81,16 +80,20 @@ def print_table(summary: Summary, title: str) -> None: elif token_mode == "unlabeled": print("result-token columns marked ?: include legacy values with unknown measurement status") aggregate_count = summary.aggregate_n - if aggregate_count: - aggregate_passes = summary.aggregate_k - lower = summary.aggregate_wilson_lo - upper = summary.aggregate_wilson_hi - rate = aggregate_passes / aggregate_count if aggregate_count else 0.0 + if aggregate_count and summary.task_mean_success is not None: + task_count = sum(task.n > 0 for task in summary.tasks.values()) print( - f"aggregate success: {aggregate_passes}/{aggregate_count} ({rate:.1%}) Wilson95 [{lower:.2f},{upper:.2f}]" + f"task-cluster success: {summary.task_mean_success:.1%} across {task_count} tasks " + f"cluster-bootstrap95 [{summary.task_cluster_lo:.2f},{summary.task_cluster_hi:.2f}]" + ) + pooled_rate = summary.aggregate_k / aggregate_count + print( + f"pooled repetition success: {summary.aggregate_k}/{aggregate_count} ({pooled_rate:.1%}) " + f"Wilson95 [{summary.aggregate_wilson_lo:.2f},{summary.aggregate_wilson_hi:.2f}]" ) else: - print("aggregate success: 0/0 (n/a; no evaluated rows)") + print("task-cluster success: n/a (no evaluated tasks)") + print("pooled repetition success: 0/0 (n/a; no evaluated rows)") print(execution_coverage_statement(summary)) print(completeness_statement(summary)) if summary.infra_errors: @@ -205,6 +208,7 @@ def build_multi_surface_table( file_rows: list[tuple[str, list[ResultRow]]], *, expected_rows_by_column: dict[str, int | None] | None = None, + run_keys_by_column: dict[str, RunKeyValidation | None] | None = None, ) -> dict[str, Any]: """Build a per-task × per-surface grid from labeled row sets. @@ -269,6 +273,7 @@ def build_multi_surface_table( column_summary = summarize( [row for task_rows in rows_by_column[column].values() for row in task_rows], expected_rows=(expected_rows_by_column or {}).get(column), + run_keys=(run_keys_by_column or {}).get(column), ) footer[column] = { "success": successes, @@ -280,6 +285,9 @@ def build_multi_surface_table( column_summary.variable_tool_tasks if column_summary.tool_distribution_available else None ), "tasks": len(rows_by_column[column]), + "task_mean_success": column_summary.task_mean_success, + "task_cluster_lo": column_summary.task_cluster_lo, + "task_cluster_hi": column_summary.task_cluster_hi, "complete": column_summary.complete, "completeness": completeness_statement(column_summary), "coverage": execution_coverage_statement(column_summary), @@ -321,7 +329,14 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) footer_parts = [] for column in columns: values = footer[column] - rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + pooled = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + if values["task_mean_success"] is None: + rate = f"task-cluster n/a; pooled {pooled}" + else: + rate = ( + f"task-cluster {values['task_mean_success']:.1%} " + f"[{values['task_cluster_lo']:.2f},{values['task_cluster_hi']:.2f}]; pooled {pooled}" + ) variability = ( f"{values['tool_variability']}/{values['tasks']} variable" if values["tool_variability"] is not None @@ -357,13 +372,19 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) lines.append("-" * len(heading)) for column in columns: values = footer[column] - rate = f"{values['success']}/{values['n']}" if values["n"] else "0/0" - percentage = f" ({100 * values['success'] / values['n']:.0f}%)" if values["n"] else "" + pooled = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + if values["task_mean_success"] is None: + rate = f"task-cluster n/a; pooled {pooled}" + else: + rate = ( + f"task-cluster {values['task_mean_success']:.1%} " + f"[{values['task_cluster_lo']:.2f},{values['task_cluster_hi']:.2f}]; pooled {pooled}" + ) variability = ( f"{values['tool_variability']}/{values['tasks']} tasks" if values["tool_variability"] is not None else "—" ) lines.append( - f"{column:12} success {rate}{percentage} total calls {values['calls']}" + f"{column:12} success {rate} total calls {values['calls']}" f" tool variability {variability} infra {values['infra_errors']}" ) for column in columns: @@ -383,23 +404,3 @@ def surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: if counts: return max(counts, key=counts.get) # type: ignore[arg-type] return path.stem - - -def warn_if_table_mixes_batteries(file_rows: list[tuple[str, list[ResultRow]]]) -> bool: - """Warn when table columns contain rows from different task batteries.""" - by_label: dict[str, set[str]] = {} - all_fingerprints: set[str] = set() - for label, rows in file_rows: - fingerprints = {read_result(row).battery or "" for row in rows if not is_meta_row(row)} - if fingerprints: - by_label[label] = fingerprints - all_fingerprints.update(fingerprints) - if len(all_fingerprints) <= 1: - return False - detail = "; ".join(f"{label}={','.join(sorted(values))}" for label, values in by_label.items()) - print( - "warning: table spans battery fingerprints; these rows were graded on " - f"different task prompts/questions and are not directly comparable ({detail})", - file=sys.stderr, - ) - return True diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py index 8db1c690..f100d3b7 100644 --- a/tests/evals/report/test_compare.py +++ b/tests/evals/report/test_compare.py @@ -90,3 +90,22 @@ def test_ab_compare_multi_rep_uses_median_successful_call_counts(): assert comparison["multi_rep"] is True assert comparison["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] + + +def test_ab_compare_excludes_trace_invalid_call_counts(): + rows_a = [ + { + "task_id": "R1", + "success": True, + "trace_integrity": False, + "trace_integrity_reason": "result_pair_mismatch", + "num_calls": 99, + "calls": [], + } + ] + rows_b = [{"task_id": "R1", "success": True, "num_calls": 1, "calls": []}] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["n_paired"] == 0 + assert comparison["paired_tasks"] == [] diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py new file mode 100644 index 00000000..f36abf19 --- /dev/null +++ b/tests/evals/report/test_identity.py @@ -0,0 +1,402 @@ +"""Persisted run-identity guards for every report path.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from evals import report as report_mod + + +def _row(task_id: str = "R1", **overrides: Any) -> dict[str, Any]: + row = { + "task_id": task_id, + "rep": 0, + "label": "candidate", + "battery": "samebattery1", + "resolved_model": "configured-model", + "provider": "anthropic", + "driver": "api", + "server": "local", + "model": "realized-model", + "requested_model": "standard", + "requested_tier": "standard", + "success": True, + "num_calls": 1, + "calls": [], + } + row.update(overrides) + return row + + +def _meta(**overrides: Any) -> dict[str, Any]: + row = { + "row_type": "meta", + "run_id": "run-1", + "battery": "samebattery1", + "resolved_model": "configured-model", + "provider": "anthropic", + "driver": "api", + "server": "local", + "model": "configured-model", + } + row.update(overrides) + return row + + +def _write(path: Path, *rows: dict[str, Any]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def _assert_refused(rc: int, capsys, detail: str) -> None: + assert rc == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "comparability cannot be established from the persisted identity" in captured.err + assert detail in captured.err + assert "aggregate success" not in captured.err + assert "A/B compare" not in captured.err + + +def test_single_summary_refuses_rows_mixing_batteries(tmp_path, capsys): + path = tmp_path / "mixed.jsonl" + _write(path, _row("R1", battery="battery-a"), _row("R2", battery="battery-b")) + + _assert_refused(report_mod.main([str(path)]), capsys, "rows disagree on battery") + + +def test_ab_report_refuses_battery_mismatch(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(battery="battery-a")) + _write(path_b, _row(battery="battery-b")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "battery differs across files") + + +def test_multi_surface_table_refuses_battery_mismatch(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(battery="battery-a")) + _write(path_b, _row(battery="battery-b")) + + rc = report_mod.main(["--table", str(path_a), str(path_b)]) + + _assert_refused(rc, capsys, "battery differs across files") + + +def test_vary_battery_is_a_usage_error(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "battery", str(path)]) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "--vary battery is not allowed" in captured.err + assert "measurement universe cannot be waived" in captured.err + + +def test_vary_all_is_a_usage_error(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "all", str(path)]) == 2 + assert "--vary has no 'all'" in capsys.readouterr().err + + +def test_vary_requires_each_dimension_to_be_named(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "provider,", str(path)]) == 2 + assert "--vary requires a dimension name" in capsys.readouterr().err + + +def test_requested_tier_cannot_be_declared_as_an_identity_dimension(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "requested_tier", str(path)]) == 2 + assert "unknown --vary dimension(s): requested_tier" in capsys.readouterr().err + + +def test_vary_resolved_model_prints_treatment_and_reports_normally(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(resolved_model="model-a", model="realized-a")) + _write(path_b, _row(resolved_model="model-b", model="realized-b")) + + rc = report_mod.main(["--vary", "resolved_model", str(path_a), str(path_b)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "Treatment: resolved_model" in captured.out + assert "Realized model evidence:" in captured.out + assert "A/B compare:" in captured.out + assert captured.err == "" + + +def test_two_varied_dimensions_print_end_to_end_attribution_label(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(resolved_model="model-a", provider="anthropic")) + _write(path_b, _row(resolved_model="model-b", provider="openai")) + + rc = report_mod.main(["--vary", "provider,resolved_model", str(path_a), str(path_b)]) + + assert rc == 0 + output = capsys.readouterr().out + assert ( + "Treatment: resolved_model, provider — end-to-end comparison; effect not attributable to any single dimension" + ) in output + + +def test_header_row_disagreement_is_refused_before_latest_wins_dedupe(tmp_path, capsys): + path = tmp_path / "resume.jsonl" + _write( + path, + _meta(), + _row(resolved_model="conflicting-model"), + _row(resolved_model="configured-model"), + ) + + _assert_refused(report_mod.main([str(path)]), capsys, "raw rows on resolved_model") + + +def test_single_header_row_identity_disagreement_is_refused(tmp_path, capsys): + path = tmp_path / "integrity.jsonl" + _write(path, _meta(), _row(provider="openai")) + + _assert_refused(report_mod.main([str(path)]), capsys, "raw rows on provider") + + +def test_conflicting_meta_headers_are_refused(tmp_path, capsys): + path = tmp_path / "headers.jsonl" + _write(path, _meta(run_id="run-1"), _meta(run_id="run-2", driver="codex-cli")) + + _assert_refused(report_mod.main([str(path)]), capsys, "conflicting meta headers for driver") + + +def test_missing_identity_value_is_not_a_wildcard(tmp_path, capsys): + path_a = tmp_path / "legacy.jsonl" + path_b = tmp_path / "identified.jsonl" + _write(path_a, _row(battery="")) + _write(path_b, _row(battery="samebattery1")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "") + + +def test_requested_tier_difference_is_not_an_identity_mismatch(tmp_path, capsys): + path_a = tmp_path / "tier.jsonl" + path_b = tmp_path / "model-id.jsonl" + _write(path_a, _row(requested_model="standard", requested_tier="standard")) + _write(path_b, _row(requested_model="configured-model", requested_tier=None)) + + assert report_mod.main([str(path_a), str(path_b)]) == 0 + captured = capsys.readouterr() + assert "A/B compare:" in captured.out + assert captured.err == "" + + +def test_meta_configured_model_is_not_compared_to_row_realized_model(tmp_path, capsys): + path = tmp_path / "api.jsonl" + _write(path, _meta(model="configured-model"), _row(model="provider-reported-model")) + + assert report_mod.main([str(path)]) == 0 + captured = capsys.readouterr() + assert "Realized model evidence:" in captured.out + assert "provider-reported-model" in captured.out + assert captured.err == "" + + +def test_resume_rows_may_have_different_run_ids(tmp_path, capsys): + path = tmp_path / "resume.jsonl" + _write(path, _meta(run_id="original"), _row("R1", run_id="original"), _row("R2", run_id="resumed")) + + assert report_mod.main([str(path)]) == 0 + assert capsys.readouterr().err == "" + + +def test_unacknowledged_provider_mismatch_is_refused(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(provider="anthropic")) + _write(path_b, _row(provider="openai")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "provider differs across files") + + +def test_realized_model_change_within_run_is_flagged_without_refusal(tmp_path, capsys): + path = tmp_path / "changed-model.jsonl" + _write(path, _row("R1", model="reported-a"), _row("R2", model="reported-b")) + + assert report_mod.main([str(path)]) == 0 + captured = capsys.readouterr() + assert "WARNING: realized model changed within" in captured.out + assert "reported-a,reported-b" in captured.out + assert captured.err == "" + + +def test_vary_driver_header_limits_claim_to_end_to_end_driver_question(tmp_path, capsys): + path_a = tmp_path / "api.jsonl" + path_b = tmp_path / "cli.jsonl" + _write(path_a, _row(driver="api")) + _write(path_b, _row(driver="codex-cli")) + + assert report_mod.main(["--vary", "driver", str(path_a), str(path_b)]) == 0 + output = capsys.readouterr().out + assert "end-to-end driver question; cannot support a surface-only claim" in output + + +def test_server_can_be_declared_as_a_treatment(tmp_path, capsys): + path_a = tmp_path / "local.jsonl" + path_b = tmp_path / "external.jsonl" + _write(path_a, _row(server="local")) + _write(path_b, _row(server="external")) + + assert report_mod.main(["--vary", "server", str(path_a), str(path_b)]) == 0 + assert "Treatment: server" in capsys.readouterr().out + + +def test_markdown_table_separates_identity_header_from_table(tmp_path, capsys): + path = tmp_path / "model.jsonl" + _write(path, _row()) + + assert report_mod.main(["--table", "--markdown", str(path)]) == 0 + output = capsys.readouterr().out + assert "Realized model evidence:" in output + assert "\n\n| task |" in output + + +def test_report_rejects_manifest_variation_within_one_result_file(tmp_path, capsys): + path = tmp_path / "manifest-changed.jsonl" + _write( + path, + _row("R1", tool_manifest_fingerprint="manifest-a"), + _row("R2", tool_manifest_fingerprint="manifest-b"), + ) + + _assert_refused( + report_mod.main([str(path)]), + capsys, + "rows disagree on tool_manifest_fingerprint", + ) + + +def test_report_identifies_but_does_not_refuse_different_tool_manifests(tmp_path, capsys): + path_a = tmp_path / "surface-a.jsonl" + path_b = tmp_path / "surface-b.jsonl" + _write(path_a, _row(tool_manifest_fingerprint="manifest-a")) + _write(path_b, _row(tool_manifest_fingerprint="manifest-b")) + + assert report_mod.main([str(path_a), str(path_b)]) == 0 + captured = capsys.readouterr() + assert "Tool manifest evidence:" in captured.out + assert "manifest-a" in captured.out + assert "manifest-b" in captured.out + assert captured.err == "" + + +def test_missing_manifest_observation_is_not_fatal(tmp_path, capsys): + path = tmp_path / "no-manifest.jsonl" + _write(path, _row(tool_manifest_fingerprint=None)) + + assert report_mod.main([str(path)]) == 0 + output = capsys.readouterr().out + assert "Summary:" in output + assert "TOOL MANIFEST ABSENT" not in output + + +def test_ab_and_table_warn_prominently_when_any_tool_manifest_is_absent(tmp_path, capsys): + path_a = tmp_path / "surface-a.jsonl" + path_b = tmp_path / "surface-b.jsonl" + _write(path_a, _row(tool_manifest_fingerprint="manifest-a")) + _write(path_b, _row(tool_manifest_fingerprint=None)) + + assert report_mod.main([str(path_a), str(path_b)]) == 0 + ab_output = capsys.readouterr().out + assert "WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified" in ab_output + assert str(path_b) in ab_output + + assert report_mod.main(["--table", str(path_a), str(path_b)]) == 0 + table_output = capsys.readouterr().out + assert "WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified" in table_output + assert str(path_b) in table_output + + +def test_exact_run_keys_name_missing_and_duplicate_rows_before_latest_wins(tmp_path, capsys): + path = tmp_path / "wrong-keys.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 1 + output = capsys.readouterr().out + assert "RUN INCOMPLETE:" in output + assert "missing keys=[R2[rep=0]]" in output + assert "unexpected keys=[R1[rep=0]]" in output + + +def test_exact_run_keys_reject_duplicate_excess_even_when_every_expected_key_exists(tmp_path, capsys): + path = tmp_path / "duplicate-excess.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1"), + _row("R2"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 1 + output = capsys.readouterr().out + assert "RUN INCOMPLETE: 2/2 rows completed" in output + assert "unexpected keys=[R1[rep=0]]" in output + + +def test_exact_run_keys_accept_append_only_retry_history_when_only_last_row_is_terminal(tmp_path, capsys): + path = tmp_path / "retry-history.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1", success=False, error="timeout", error_class="infra_cli"), + _row("R2"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 0 + output = capsys.readouterr().out + assert "RUN COMPLETE: 2/2 rows completed" in output + + +def test_exact_run_keys_accept_declared_task_subset_with_all_repetitions(tmp_path, capsys): + path = tmp_path / "subset.jsonl" + _write( + path, + _meta(expected_rows=4, expected_task_ids=["R1", "W1"], expected_reps=2), + _row("R1", rep=0), + _row("R1", rep=1), + _row("W1", rep=0), + _row("W1", rep=1), + ) + + assert report_mod.main([str(path)]) == 0 + assert "RUN COMPLETE: 4/4 rows completed" in capsys.readouterr().out + + +def test_malformed_exact_run_expectation_is_refused_instead_of_treated_as_legacy(tmp_path, capsys): + path = tmp_path / "malformed-expectation.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R1"], expected_reps=1), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "invalid run expectation" in captured.err + assert "expected_task_ids contains duplicates" in captured.err diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 85c221b0..2e7e0e00 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -26,7 +26,7 @@ def test_completeness_is_independent_from_success_rate(): [ {"task_id": "R1", "success": True, "calls": []}, {"task_id": "R2", "success": False, "calls": []}, - {"task_id": "C1", "skipped": "env:plan-gated:customers", "calls": []}, + {"task_id": "L4", "skipped": "env:plan-gated:customers", "calls": []}, ], expected_rows=3, ) @@ -37,7 +37,7 @@ def test_completeness_is_independent_from_success_rate(): assert complete.complete is True assert completeness_statement(complete).startswith("RUN COMPLETE:") assert execution_coverage_statement(complete) == ( - "EXECUTION COVERAGE: 2/3 rows evaluated (66.7%); skipped tasks=[C1 (env:plan-gated:customers)]" + "EXECUTION COVERAGE: 2/3 rows evaluated (66.7%); skipped tasks=[L4 (env:plan-gated:customers)]" ) missing_worker = summarize( @@ -93,6 +93,30 @@ def test_completeness_is_independent_from_success_rate(): assert cleanup.complete is False +def test_result_pair_mismatch_preserves_outcome_but_excludes_trace_metrics(): + summary = summarize( + [ + { + "task_id": "R1", + "success": True, + "trace_integrity": False, + "trace_integrity_reason": "result_pair_mismatch", + "result_pair_mismatch": True, + "num_calls": 99, + "calls": [{"tool": "untrustworthy", "result_tokens": 200}], + } + ], + expected_rows=1, + ) + + assert summary.complete is False + assert summary.trace_invalid_rows == 1 + assert summary.aggregate_k == summary.aggregate_n == 1 + assert summary.tasks["R1"].med_calls is None + assert summary.tasks["R1"].tool_reps == 0 + assert summary.tasks["R1"].med_result_tokens is None + + def _summarize_excludes_infra_errors_from_success(): rows = [ {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, @@ -250,6 +274,26 @@ def test_wilson_interval_bounds(): assert wilson_interval(0, 0) == (0.0, 0.0) +def test_headline_interval_bootstraps_35_task_clusters_instead_of_175_repetitions(): + rows = [ + {"task_id": f"T{task_index:02d}", "rep": rep, "success": task_index < 17, "calls": []} + for task_index in range(35) + for rep in range(5) + ] + + summary = summarize(rows) + + assert (summary.aggregate_k, summary.aggregate_n) == (85, 175) + assert summary.aggregate_wilson_lo == pytest.approx(0.4127693534) + assert summary.aggregate_wilson_hi == pytest.approx(0.5592729455) + assert summary.task_mean_success == pytest.approx(17 / 35) + assert summary.task_cluster_lo == pytest.approx(0.3142857143) + assert summary.task_cluster_hi == pytest.approx(0.6571428571) + assert summary.task_cluster_hi - summary.task_cluster_lo > ( + summary.aggregate_wilson_hi - summary.aggregate_wilson_lo + ) + + def test_multi_rep_synthetic_file_reports_wilson_and_instability_without_noise_claim(tmp_path: Path, capsys): path = tmp_path / "multi.jsonl" outcomes = { @@ -302,7 +346,8 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): assert capsys.readouterr().out == ( "Summary: sample.jsonl\n" - "aggregate success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" + "task-cluster success: 100.0% across 1 tasks cluster-bootstrap95 [1.00,1.00]\n" + "pooled repetition success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" "RUN COMPLETE: 1/1 rows completed\n" "tool variability: —\n" diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 408d5d58..29b53562 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -66,7 +66,7 @@ def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_pat collision.write_text( "\n".join( [ - json.dumps({"row_type": "meta", "expected_rows": 1}), + json.dumps({"row_type": "meta", "expected_rows": 1, "server": "local"}), json.dumps(_synth_row("R1", skipped="env:fixture-collision:customers:Acme")), ] ) @@ -76,7 +76,7 @@ def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_pat assert report_mod.main([str(collision)]) == 1 collision_output = capsys.readouterr().out - assert "aggregate success: 0/0" in collision_output + assert "pooled repetition success: 0/0" in collision_output assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in collision_output assert "R1 (env:fixture-collision:customers:Acme)" in collision_output assert "RUN INCOMPLETE:" in collision_output @@ -86,8 +86,8 @@ def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_pat plan_gated.write_text( "\n".join( [ - json.dumps({"row_type": "meta", "expected_rows": 1}), - json.dumps(_synth_row("C1", skipped="env:plan-gated:customers")), + json.dumps({"row_type": "meta", "expected_rows": 1, "server": "local"}), + json.dumps(_synth_row("L4", skipped="env:plan-gated:customers")), ] ) + "\n", @@ -96,9 +96,9 @@ def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_pat assert report_mod.main([str(plan_gated)]) == 0 plan_output = capsys.readouterr().out - assert "aggregate success: 0/0" in plan_output + assert "pooled repetition success: 0/0" in plan_output assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in plan_output - assert "C1 (env:plan-gated:customers)" in plan_output + assert "L4 (env:plan-gated:customers)" in plan_output assert "RUN COMPLETE:" in plan_output assert "expected skips=1 [plan-gated=1]" in plan_output @@ -172,7 +172,7 @@ def _report_main_table_cli(tmp_path, capsys): assert "R1" in out -def _report_main_table_warns_when_battery_fingerprints_differ(tmp_path, capsys): +def _report_main_table_refuses_when_battery_fingerprints_differ(tmp_path, capsys): f1 = tmp_path / "old.jsonl" f2 = tmp_path / "new.jsonl" f1.write_text( @@ -186,10 +186,11 @@ def _report_main_table_warns_when_battery_fingerprints_differ(tmp_path, capsys): rc = report_mod.main(["--table", str(f1), str(f2)]) - assert rc == 0 + assert rc == 2 captured = capsys.readouterr() - assert "spans battery fingerprints" in captured.err - assert "different task prompts/questions" in captured.err + assert captured.out == "" + assert "comparability cannot be established from the persisted identity" in captured.err + assert "battery differs across files" in captured.err def _report_main_markdown_flag(tmp_path, capsys): @@ -198,7 +199,8 @@ def _report_main_markdown_flag(tmp_path, capsys): rc = report_mod.main(["--table", "--markdown", str(f1)]) assert rc == 0 out = capsys.readouterr().out - assert out.startswith("| task |") + assert out.startswith("WARNING: TOOL MANIFEST ABSENT") + assert "\n\n| task |" in out assert "| R1 |" in out assert "---" in out @@ -226,7 +228,7 @@ def _report_main_no_dedupe_flag(tmp_path, capsys): _report_marks_entirely_estimated_result_token_columns, _report_marks_mixed_measured_and_estimated_columns, _report_main_table_cli, - _report_main_table_warns_when_battery_fingerprints_differ, + _report_main_table_refuses_when_battery_fingerprints_differ, _report_main_markdown_flag, _report_main_no_dedupe_flag, ), @@ -331,7 +333,8 @@ def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): "-------------------------------------------------------\n" "R1 In project P, what is the curren… ✅ 2c · tools —\n" "-------------------------------------------------------\n" - "local success 1/1 (100%) total calls 2 tool variability — infra 0\n" + "local success task-cluster 100.0% [1.00,1.00]; pooled 1/1 total calls 2 " + "tool variability — infra 0\n" "local EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" "local RUN COMPLETE: 1/1 rows completed\n" ) From 031037fef5c567279453091f1920f3434f112835 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 20:57:00 +0530 Subject: [PATCH 40/93] Give each repetition its own fixture seed, and record it The per-repetition seed was a throwaway uuid that was never persisted, while the randomisation docstring claimed the opposite - so a failed fixture had never actually been reproducible. Each repetition now carries a fixture_seed_id that is persisted on its row and drives its hidden truth, and the docstring names the field. Keeping the seed per-repetition also makes a cross-repetition leak impossible by construction rather than by guard: knowing one repetition's seed reveals nothing about the next. Persisted seed forensics record which namespaces were randomised, never the values or the seeded entity ids. A capability skip is now bound to the tasks whose fixtures can actually be gated on it, derived from the same fixture needs the collision categories use, so an unrelated task can no longer skip its way to a complete run. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/build.py | 5 +++ evals/seed/customers.py | 3 ++ evals/seed/cycles.py | 3 ++ evals/seed/identities.py | 30 +++++++++++++++++ evals/seed/intake.py | 5 +++ evals/seed/item_types.py | 3 ++ evals/seed/labels.py | 3 ++ evals/seed/modules.py | 3 ++ evals/seed/projects.py | 10 +++++- evals/seed/randomize.py | 18 ++++++----- evals/seed/releases.py | 2 ++ evals/seed/states.py | 2 ++ evals/seed/work_items.py | 9 +++++- evals/skip_taxonomy.py | 36 +++++++++++++++++---- evals/tasks/__init__.py | 4 +++ evals/tasks/answers.py | 5 +-- evals/tasks/catalog.py | 34 ++++++++++++------- evals/tasks/read.py | 4 +-- tests/evals/seed/test_read_randomization.py | 4 +++ tests/evals/tasks/test_catalog.py | 32 +++++++++++++++++- tests/evals/tasks/test_output_contracts.py | 2 ++ 21 files changed, 183 insertions(+), 34 deletions(-) create mode 100644 evals/seed/identities.py diff --git a/evals/seed/build.py b/evals/seed/build.py index c2bfe2cc..a695074d 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -16,6 +16,7 @@ seed_customer, ) from .cycles import seed_cycles +from .identities import record_seeded_entity from .intake import seed_intake from .item_types import ( BUG_TYPE_NAME, @@ -311,8 +312,11 @@ def seed( "w6_unfinished_titles": list(UNFINISHED_CYCLE_TITLES), "workspace_objects": [], # [{kind, id}, ...] surviving project delete "randomized_truth": {}, + "randomized_truth_namespaces": set(), + "seeded_entity_kinds": set(), "evidence_sentinels": {}, "evidence_targets": {}, + "evidence_aggregates": {}, # None means the category was unavailable and name-based teardown must fail closed. "workspace_baseline": dict.fromkeys(_WORKSPACE_BASELINE_CATEGORIES), } @@ -331,6 +335,7 @@ def seed( initial_suffix=run_prefix.upper(), ) ctx["project_id"] = project.id + record_seeded_entity(ctx, "project", project.id) ctx["project_identifier"] = getattr(project, "identifier", None) # Workspace first, then project. Seeding is per task-rep, so S5 turning workspace diff --git a/evals/seed/customers.py b/evals/seed/customers.py index 4265e9c2..a34200a9 100644 --- a/evals/seed/customers.py +++ b/evals/seed/customers.py @@ -14,6 +14,7 @@ is_evaluation_customer_name, ) +from .identities import record_seeded_entity from .projects import plan_gate_skips __all__ = [ @@ -33,6 +34,7 @@ def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, An data=CreateCustomer(name=CUSTOMER_NAME), ) context["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} + record_seeded_entity(context, "customer", customer.id) context["workspace_objects"].append({"kind": "customer", "id": customer.id}) request = plane.customers.requests.create( workspace_slug=workspace_slug, @@ -44,3 +46,4 @@ def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, An "name": CUSTOMER_REQUEST_NAME, "customer_id": customer.id, } + record_seeded_entity(context, "customer_request", request.id) diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py index 5cb0b004..16f171e1 100644 --- a/evals/seed/cycles.py +++ b/evals/seed/cycles.py @@ -12,6 +12,7 @@ from evals.evidence import set_target_evidence from evals.fixtures import CYCLE_CURRENT, CYCLE_PAST, PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES +from .identities import record_seeded_entity from .randomize import random_truth_rng, record_randomized_truth @@ -94,6 +95,8 @@ def seed_cycles( past_name: past.id, current_name: current.id, } + record_seeded_entity(context, "cycle", past.id) + record_seeded_entity(context, "cycle", current.id) context["cycle_past_name"] = past_name context["cycle_current_name"] = current_name context["cycle_past_id"] = past.id diff --git a/evals/seed/identities.py b/evals/seed/identities.py new file mode 100644 index 00000000..49aee301 --- /dev/null +++ b/evals/seed/identities.py @@ -0,0 +1,30 @@ +"""Non-secret seed-shape metadata safe to persist beside evaluation results.""" + +from __future__ import annotations + +from typing import Any + + +def record_seeded_entity(context: dict[str, Any], kind: str, object_id: Any) -> None: + """Register that a fixture kind was seeded without retaining its target id.""" + value = str(object_id or "").strip() + if not value: + return + context.setdefault("seeded_entity_kinds", set()).add(str(kind)) + + +def capture_seed_artifacts(context: dict[str, Any]) -> tuple[list[str], list[str]]: + """Copy only fixture kinds and randomization namespaces, never ids or truth values.""" + entity_kinds = context.get("seeded_entity_kinds") + randomized_namespaces = context.get("randomized_truth_namespaces") + return ( + sorted({str(kind) for kind in entity_kinds}) if isinstance(entity_kinds, (list, set, tuple)) else [], + ( + sorted({str(namespace) for namespace in randomized_namespaces}) + if isinstance(randomized_namespaces, (list, set, tuple)) + else [] + ), + ) + + +__all__ = ["capture_seed_artifacts", "record_seeded_entity"] diff --git a/evals/seed/intake.py b/evals/seed/intake.py index 78cc5ff5..66d23c0c 100644 --- a/evals/seed/intake.py +++ b/evals/seed/intake.py @@ -9,6 +9,8 @@ from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE +from .identities import record_seeded_entity + def seed_intake(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: project_id = context["project_id"] @@ -39,3 +41,6 @@ def seed_intake(plane: PlaneClient, workspace_slug: str, context: dict[str, Any] "title": INTAKE_SPAM_TITLE, }, } + for row in (billing, spam): + record_seeded_entity(context, "intake", row.id) + record_seeded_entity(context, "work_item", getattr(row, "issue", None) or row.id) diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py index 1e9fc8a3..1643e12a 100644 --- a/evals/seed/item_types.py +++ b/evals/seed/item_types.py @@ -7,6 +7,7 @@ from plane import PlaneClient from plane.models.work_item_types import CreateWorkItemType +from .identities import record_seeded_entity from .projects import is_plan_gate BUG_TYPE_NAME = "Bug" @@ -98,6 +99,7 @@ def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, A work_item_type_ids=[existing.id], ) context["bug_type"] = {"id": existing.id, "name": target} + record_seeded_entity(context, "work_item_type", existing.id) context["bug_type_created"] = created context["bug_type_workspace_level"] = True if created: @@ -122,6 +124,7 @@ def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, A ) created = True context["bug_type"] = {"id": existing.id, "name": target} + record_seeded_entity(context, "work_item_type", existing.id) context["bug_type_created"] = created context["bug_type_workspace_level"] = False except Exception as exc: diff --git a/evals/seed/labels.py b/evals/seed/labels.py index 410744fa..8ee9b017 100644 --- a/evals/seed/labels.py +++ b/evals/seed/labels.py @@ -7,6 +7,8 @@ from plane import PlaneClient from plane.models.labels import CreateLabel +from .identities import record_seeded_entity + LABEL_NAMES = ("auth", "triage", "perf") @@ -18,3 +20,4 @@ def seed_labels(plane: PlaneClient, workspace_slug: str, context: dict[str, Any] data=CreateLabel(name=name), ) context["labels"][name] = label.id + record_seeded_entity(context, "label", label.id) diff --git a/evals/seed/modules.py b/evals/seed/modules.py index 5c20c425..050f7e07 100644 --- a/evals/seed/modules.py +++ b/evals/seed/modules.py @@ -10,6 +10,7 @@ from evals.fixtures import MODULE_COMPLETED_TITLES, MODULE_NAME +from .identities import record_seeded_entity from .work_items import find_completed_state, list_states @@ -26,6 +27,7 @@ def seed_module(plane: PlaneClient, workspace_slug: str, context: dict[str, Any] data=CreateModule(name=MODULE_NAME, status="in-progress"), ) context["module"] = {"id": module.id, "name": MODULE_NAME} + record_seeded_entity(context, "module", module.id) completed_ids: list[str] = [] for title in MODULE_COMPLETED_TITLES: item = plane.work_items.create( @@ -46,6 +48,7 @@ def seed_module(plane: PlaneClient, workspace_slug: str, context: dict[str, Any] completed_ids.append(item.id) context["items"][title] = item.id context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) plane.modules.add_work_items( workspace_slug=workspace_slug, project_id=project_id, diff --git a/evals/seed/projects.py b/evals/seed/projects.py index b8ce65f4..9cb29dd4 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -14,8 +14,9 @@ from plane.models.workspaces import WorkspaceFeature from evals.errors import TaskSkipped -from evals.evidence import set_target_evidence +from evals.evidence import set_target_evidence, set_target_grouped_count_evidence +from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth # Plane's project identifier field is capped at 12 characters. Keep two characters @@ -232,6 +233,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s initial_suffix=run_prefix.upper(), ) context["second_project_id"] = project.id + record_seeded_entity(context, "project", project.id) context["second_project_name"] = name context["second_project_identifier"] = getattr(project, "identifier", None) enable_project_features(plane, workspace_slug, project.id) @@ -285,6 +287,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s main_bug_ids.append(item.id) context["items"][title] = item.id context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) second_bug_ids: list[str] = [] for title in second_titles: item = plane.work_items.create( @@ -293,6 +296,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] ) second_bug_ids.append(item.id) + record_seeded_entity(context, "work_item", item.id) if task_id != "R6": context["r6_main_bug_count"] = len(main_bug_ids) context["r6_second_bug_count"] = len(second_bug_ids) @@ -333,3 +337,7 @@ def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: "winner": context["r6_more_bugs_project"], } set_target_evidence(context, [*main_titles, *second_titles], target_ids=[main_id, project.id]) + set_target_grouped_count_evidence( + context, + {main_id: confirmed_main, str(project.id): confirmed_second}, + ) diff --git a/evals/seed/randomize.py b/evals/seed/randomize.py index 855a8462..4c34fabf 100644 --- a/evals/seed/randomize.py +++ b/evals/seed/randomize.py @@ -8,11 +8,12 @@ def random_truth_rng(context: dict[str, Any], namespace: str) -> random.Random: - """Return a reproducible RNG keyed by the full private run id and a namespace. + """Return a reproducible RNG keyed by the per-repetition fixture seed and a namespace. - Only the run-id prefix appears in the project name shown to the agent. The full id is - persisted on the result row, making a failed fixture reproducible without making its - hidden choices derivable from the prompt. + The caller supplies the repetition's private fixture seed as ``context["run_id"]``. + Its full value is persisted explicitly as ``TaskResult.fixture_seed_id``, making a + failed fixture reproducible without making later repetitions' independent choices + derivable from this one. """ run_id = str(context.get("run_id") or "") if not run_id: @@ -22,11 +23,11 @@ def random_truth_rng(context: dict[str, Any], namespace: str) -> random.Random: def random_truth_token(context: dict[str, Any], namespace: str, *, length: int = 10) -> str: - """Return a reproducible hidden token derived from the full private run id. + """Return a reproducible hidden token derived from the per-repetition fixture seed. Unlike the visible eight-character project prefix, this token depends on the full - run id and a task namespace. It gives response evidence a realistically unique value - without making failed fixture reproduction nondeterministic. + fixture seed id and a task namespace. It gives response evidence a realistically unique + value without making failed fixture reproduction nondeterministic. """ run_id = str(context.get("run_id") or "") if not run_id: @@ -37,8 +38,9 @@ def random_truth_token(context: dict[str, Any], namespace: str, *, length: int = def record_randomized_truth(context: dict[str, Any], key: str, value: Any) -> None: - """Retain the chosen hidden value in seed context for diagnostics.""" + """Retain hidden truth in memory and separately register its persistable namespace.""" context.setdefault("randomized_truth", {})[key] = value + context.setdefault("randomized_truth_namespaces", set()).add(str(key)) __all__ = ["random_truth_rng", "random_truth_token", "record_randomized_truth"] diff --git a/evals/seed/releases.py b/evals/seed/releases.py index ca2f6cff..8470d8b5 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -15,6 +15,7 @@ RELEASE_NAME, ) +from .identities import record_seeded_entity from .projects import plan_gate_skips from .randomize import random_truth_rng, random_truth_token, record_randomized_truth @@ -54,6 +55,7 @@ def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any raise RuntimeError("release create response did not confirm the randomized release name") confirmed_release_name = confirmed_release_name or release_name context["release"] = {"id": release.id, "name": confirmed_release_name} + record_seeded_entity(context, "release", release.id) context["release_name"] = confirmed_release_name context["workspace_objects"].append({"kind": "release", "id": release.id}) # Single changelog body; DESIGN's "2 entries" are encoded as plain text. diff --git a/evals/seed/states.py b/evals/seed/states.py index 0c03fcce..355bf69c 100644 --- a/evals/seed/states.py +++ b/evals/seed/states.py @@ -10,6 +10,7 @@ from evals.evidence import set_target_evidence from evals.state_oracle import state_name_group_pairs +from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth @@ -43,6 +44,7 @@ def seed_r7_state_oracle(plane: PlaneClient, workspace_slug: str, context: dict[ ) context["r7_state_pairs"] = pairs context["r7_random_state_id"] = created_id + record_seeded_entity(context, "state", created_id) record_randomized_truth( context, "R7.states", diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index cda2dc0d..5bd7af1a 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -13,7 +13,7 @@ from evals.changelog import normalize_changelog_text from evals.errors import TaskSkipped -from evals.evidence import set_target_evidence +from evals.evidence import set_target_count_evidence, set_target_evidence from evals.fixtures import ( BLOCKING_REFERENCE_ADDRESS, BLOCKING_SOURCE_TITLE, @@ -28,6 +28,7 @@ WORK_ITEM_FIXTURES, ) +from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth __all__ = [ @@ -172,6 +173,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, if not getattr(random_state, "id", None): raise RuntimeError(f"seed {task_id}: random state create returned no id") state_targets[PAYMENT_WEBHOOK_TITLE if task_id == "R1" else SIDEBAR_TITLE] = random_state + record_seeded_entity(context, "state", random_state.id) record_randomized_truth( context, f"{task_id}.state", @@ -239,6 +241,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, context["fixture_item_ids"][fixture_title] = item.id context["fixture_item_titles"][fixture_title] = title context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) sequence = getattr(item, "sequence_id", None) if sequence is not None and context.get("project_identifier"): context["item_identifiers"][fixture_title] = f"{context['project_identifier']}-{sequence}" @@ -275,6 +278,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, ) if getattr(created_comment, "id", None) is not None: comment_ids.add(str(created_comment.id)) + record_seeded_entity(context, "comment", created_comment.id) # Capture every affected read oracle from API-confirmed state, never from the random choice. if task_id in {"R1", "I2"}: @@ -300,6 +304,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, context["r2_urgent_open_count"] = len(confirmed_titles) context["randomized_truth"]["R2.urgent_open_count"]["confirmed"] = len(confirmed_titles) set_target_evidence(context, confirmed_titles, target_ids=[project_id]) + set_target_count_evidence(context, len(confirmed_titles), target_ids=[project_id]) if task_id == "R3": confirmed_due_titles: list[str] = [] @@ -385,6 +390,8 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, work_item_id=attachment_target_id, ) confirmed_rows = list(attachments.results or []) + for attachment in confirmed_rows: + record_seeded_entity(context, "attachment", getattr(attachment, "id", None)) confirmed_attachment_count = len(confirmed_rows) context["l5_attachment_count"] = confirmed_attachment_count context["randomized_truth"]["L5.attachment_count"]["confirmed"] = confirmed_attachment_count diff --git a/evals/skip_taxonomy.py b/evals/skip_taxonomy.py index 22ea32b3..18eba332 100644 --- a/evals/skip_taxonomy.py +++ b/evals/skip_taxonomy.py @@ -32,6 +32,16 @@ } ) +# Task eligibility is derived from the fixture ``needs`` in the task catalog. This +# maps fixture groups to the one capability refusal that their seeder can emit; it +# deliberately does not duplicate a task-id table. +_PLAN_GATED_CAPABILITY_BY_NEED = { + "bug_type": "work-item-types", + "customer": "customers", + "release": "releases", +} +_ACTIVITY_WORKER_NEED = "activity_feed" + def _plan_gated_capability(reason: str) -> str | None: if not reason.startswith(PLAN_GATED_PREFIX): @@ -40,20 +50,34 @@ def _plan_gated_capability(reason: str) -> str | None: return capability if capability in PLAN_GATED_CAPABILITIES else None -def classify_skip_reason(reason: str) -> SkipDisposition: +def _task_expected_capability_reasons(task_id: str) -> frozenset[str]: + from evals.tasks import TASKS_BY_ID + + task = TASKS_BY_ID.get(task_id) + needs = set(task.get("needs") or ()) if task is not None else set() + reasons = { + f"{PLAN_GATED_PREFIX}{capability}" + for need, capability in _PLAN_GATED_CAPABILITY_BY_NEED.items() + if need in needs + } + if _ACTIVITY_WORKER_NEED in needs: + reasons.add(NO_ACTIVITY_WORKER_REASON) + return frozenset(reasons) + + +def classify_skip_reason(reason: str, *, task_id: str | None = None) -> SkipDisposition: """Classify a known capability skip, dirty environment, or unknown reason.""" - if _plan_gated_capability(reason) is not None: - return "expected-capability" - if reason == NO_ACTIVITY_WORKER_REASON: + is_known_capability = _plan_gated_capability(reason) is not None or reason == NO_ACTIVITY_WORKER_REASON + if is_known_capability and (task_id is None or reason in _task_expected_capability_reasons(task_id)): return "expected-capability" if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): return "dirty-environment" return "unexpected" -def is_expected_environment_capability_skip(reason: str) -> bool: +def is_expected_environment_capability_skip(reason: str, *, task_id: str | None = None) -> bool: """Return whether a known absent environment capability caused the skip.""" - return classify_skip_reason(reason) == "expected-capability" + return classify_skip_reason(reason, task_id=task_id) == "expected-capability" def skip_reason_family(reason: str) -> str: diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index 7c3fe368..9a1e6d96 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -18,6 +18,8 @@ battery_fingerprint, get_tasks, task_author, + task_fingerprint, + task_fingerprint_payload, ) from evals.tasks.cross import verify_c1, verify_c2 from evals.tasks.debias import ( @@ -94,6 +96,8 @@ "state_group", "state_name", "task_author", + "task_fingerprint", + "task_fingerprint_payload", "whole_answer_int", "word_boundary", "I1_TITLE", diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py index d7a66389..81f57797 100644 --- a/evals/tasks/answers.py +++ b/evals/tasks/answers.py @@ -155,10 +155,7 @@ def answer_with_provenance( """ calls = run.get("calls") source = str(run.get("call_source") or "unknown") - driver_notes = run.get("driver_notes") - trace_incomplete = isinstance(driver_notes, list) and any( - isinstance(note, str) and note.startswith("proxy_sidecar_incomplete") for note in driver_notes - ) + trace_incomplete = run.get("trace_integrity") is False available = bool(run.get("evidence_trace_available")) provenance = not trace_incomplete and has_response_evidence(run) if trace_incomplete: diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 2a481816..cfcc4ced 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -96,6 +96,25 @@ def task_author(task: dict[str, Any]) -> str: """ +def task_fingerprint_payload(task: dict[str, Any]) -> dict[str, Any]: + """Return the canonical question payload shared by task and battery hashes.""" + return { + "id": task.get("id"), + "prompt": task.get("prompt"), + "needs": sorted(task.get("needs") or []), + } + + +def _short_fingerprint(document: Any) -> str: + blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + +def task_fingerprint(task: dict[str, Any]) -> str: + """Return a stable short hash of one task's own question payload.""" + return _short_fingerprint(task_fingerprint_payload(task)) + + def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: """Stable short hash of the revision and each task's ID, prompt and fixture names. @@ -106,18 +125,9 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: from the full catalog. """ src = list(TASKS if tasks is None else tasks) - payload: list[dict[str, Any]] = [] - for t in sorted(src, key=lambda x: str(x.get("id") or "")): - payload.append( - { - "id": t.get("id"), - "prompt": t.get("prompt"), - "needs": sorted(t.get("needs") or []), - } - ) + payload = [task_fingerprint_payload(task) for task in sorted(src, key=lambda item: str(item.get("id") or ""))] document = {"revision": CATALOG_REVISION, "tasks": payload} - blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + return _short_fingerprint(document) __all__ = [ @@ -127,4 +137,6 @@ def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: "battery_fingerprint", "get_tasks", "task_author", + "task_fingerprint", + "task_fingerprint_payload", ] diff --git a/evals/tasks/read.py b/evals/tasks/read.py index 8ddb0c83..03ae80fb 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -46,7 +46,7 @@ async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R2: final text reports the API-confirmed seed count with call provenance.""" + """R2: report the seed count with item-value or target-scoped total-count evidence.""" expected = ctx.get("r2_urgent_open_count") if not isinstance(expected, int): return answer_with_provenance(False, "API-confirmed urgent-open seed count missing", run) @@ -179,7 +179,7 @@ async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: - """R6: final text reports the winning project via ``project: NAME``.""" + """R6: report the winner with item-value or exact project-grouped-count evidence.""" expected = str(ctx.get("r6_more_bugs_project") or "") if not expected: return answer_with_provenance(False, "API-confirmed R6 winner missing from seed ctx", run) diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index 331f67ce..f6adb68b 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -179,6 +179,7 @@ def test_work_item_read_truth_is_randomized_and_api_confirmed(task_id, oracle_ke assert ctx[oracle_key] not in (None, "", []) assert "confirmed" in ctx["randomized_truth"][truth_key] assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] def test_r2_randomized_counts_differ_between_rows_after_api_readback(): @@ -213,6 +214,7 @@ def test_r4_cycle_inventory_is_randomized_and_api_confirmed(): "overdue_titles": ctx["r4_overdue_titles"], } assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): @@ -228,6 +230,7 @@ def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): truth = ctx["randomized_truth"]["R7.states"] assert truth["confirmed"] == ctx["r7_state_pairs"] assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] def test_l1_seed_oracle_is_the_api_confirmed_target_id(): @@ -238,3 +241,4 @@ def test_l1_seed_oracle_is_the_api_confirmed_target_id(): assert ctx["l1_expected_summary_ids"] == [ctx["fixture_item_ids"]["Payment webhook drops retries"]] assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == tuple(ctx["l1_expected_summary_ids"]) + assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] == (ctx["project_id"],) diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 9c97dede..076f1ac7 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -2,12 +2,22 @@ from __future__ import annotations +import hashlib import inspect +import json import pytest from evals import tasks as tasks_mod -from evals.tasks.catalog import TASKS, TASKS_BY_ID, battery_fingerprint, get_tasks, task_author +from evals.tasks.catalog import ( + TASKS, + TASKS_BY_ID, + battery_fingerprint, + get_tasks, + task_author, + task_fingerprint, + task_fingerprint_payload, +) from evals.tasks.debias import ( I1_TITLE, ) @@ -267,6 +277,26 @@ def test_revision_bump_changes_the_fingerprint_for_an_unchanged_catalog(): assert battery_fingerprint(tasks) == before, "restoring the revision must restore it" +def test_task_and_battery_fingerprints_share_the_same_per_task_payload(monkeypatch): + from evals.tasks import catalog + + task = {"id": "T1", "prompt": "prompt", "needs": {"projects", "states"}} + expected_payload = {"id": "T1", "prompt": "prompt", "needs": ["projects", "states"]} + assert task_fingerprint_payload(task) == expected_payload + + def short_hash(document): + blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + assert task_fingerprint(task) == short_hash(expected_payload) + sentinel_payload = {"id": "sentinel", "prompt": "shared", "needs": []} + monkeypatch.setattr(catalog, "task_fingerprint_payload", lambda _task: sentinel_payload) + assert catalog.task_fingerprint(task) == short_hash(sentinel_payload) + assert catalog.battery_fingerprint([task]) == short_hash( + {"revision": catalog.CATALOG_REVISION, "tasks": [sentinel_payload]} + ) + + def test_fingerprint_records_the_revision_transition(): """Pin the current value, so a future change is read as intentional, not drift. diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py index a0d7c4b7..1bead89b 100644 --- a/tests/evals/tasks/test_output_contracts.py +++ b/tests/evals/tasks/test_output_contracts.py @@ -254,6 +254,8 @@ async def _go(): { **_run("count: 4"), "driver_notes": ["proxy_sidecar_incomplete:skipped_rows=1"], + "trace_integrity": False, + "trace_integrity_reason": "recorder_loss", }, ) assert incomplete_ok is False From da12a35aae734afb8208ea1714131e2c473bf202 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 20:57:00 +0530 Subject: [PATCH 41/93] Make trace loss visible in the run verdict, and validate the exact key set A lost call reached no verdict: incompleteness was a prose driver note, consumed only by provenance-bearing tasks, and Summary.complete read none of it. A run whose proxy dropped calls reported RUN COMPLETE. Trace integrity is now a typed field that becomes infra_trace or infra_protocol, so the row leaves the success denominator and the run is incomplete - harness data loss is infrastructure, not an agent failure. result_pair_mismatch is exempt from that promotion: a duplicate model-generated tool id can be an agent or provider fault, and flattening it into infra_ would misfile the cause. Completeness compared counts, so a run that executed one task twice and another never still balanced. The meta header now records the task list and repetition count, and the exact (task_id, rep) set is required, named rather than counted. Resume stays append-only: a duplicate key is accepted only when every occurrence but the last is retryable. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 51 +++++-- evals/README.md | 33 +++-- evals/result_lifecycle.py | 24 ++++ evals/results.py | 65 ++++++++- evals/runner/live.py | 74 +++++++++- evals/runner/meta.py | 17 +++ evals/runner/resume.py | 14 +- tests/evals/drivers/test_api_driver.py | 153 ++++++++++++++++++++- tests/evals/drivers/test_cli_driver.py | 179 ++++++++++++++++++++++++- tests/evals/runner/test_live.py | 165 ++++++++++++++++++++--- tests/evals/runner/test_resume.py | 43 +++++- tests/evals/test_results.py | 22 +++ tests/evals/test_skip_taxonomy.py | 10 ++ 13 files changed, 775 insertions(+), 75 deletions(-) create mode 100644 evals/result_lifecycle.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 4f4f64c7..661c9c2e 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -24,10 +24,12 @@ less text but fails the task is not an improvement. Conversely, success rate alo extra calls, variable routes, and large responses. The harness therefore records all four dimensions for the same task execution. -The point is empirical comparison. Given the same task battery, model, and repetitions, -different surfaces can be compared from observed behavior rather than from tool counts, -schema inspection, or projected costs. The battery fingerprint records task IDs and prompts -so incompatible batteries are not silently compared. +The point is empirical comparison. Given the same task battery and declared treatment +dimensions, different surfaces can be compared from observed behavior rather than from tool +counts, schema inspection, or projected costs. Report identity validation refuses incompatible +batteries and undeclared canonical-identity differences before printing measurements. The +battery fingerprint records the selected task universe; per-task fingerprints preserve the +task-local payload needed for possible future intersection comparisons. ## What is measured @@ -70,6 +72,11 @@ instances under the two labels, independent tasks, and exchangeable A/B labels u permutation null; they do not account for shared environment drift or dependence between tasks. The report prints the paired task count so small samples remain visible. +Single-run success headlines use the same sampling unit: each evaluated task contributes +its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. +The pooled repetition rate and its Wilson interval remain visible as a descriptive figure, +but are labeled pooled rather than presented as the headline confidence interval. + Client-local tools such as shell or tool-search helpers are retained separately as `client_tool_calls`; they do not count as Plane calls. For an external server launched with `--server-cmd`, the runner marks the row server as `external`; call counts and observed tool @@ -117,12 +124,20 @@ make sidecars larger. The character-derived estimate remains useful for surface because it is deterministic and monotonic in the recorded response size. Read-task provenance does not turn payload recording back on. Each read seeder registers a -hidden, per-run target-entity sentinel. At the API loop or CLI recording proxy, the harness -matches that sentinel while the successful response is in memory and persists only a -non-sensitive `observed_sentinels` label. A successful unrelated Plane call therefore does -not count as evidence, and neither the response body nor the sentinel value enters the -payload-free result row. A driver path without response matching is reported as unavailable -and cannot pass a read verifier. +hidden, per-run sentinel and its seeded target entity ID. At the API loop or CLI recording +proxy, the harness requires the request arguments to target that ID and matches the sentinel +while the successful response is in memory, persisting only a non-sensitive +`observed_sentinels` label. CLI proxies receive only target IDs plus sentinel lengths and +SHA-256 fingerprints through a mode-0600 one-shot file outside the agent cwd; the raw value +is absent even from that file. They consume and unlink it before starting Plane MCP. A successful response from +an unrelated entity therefore does not count, and neither the response body nor sentinel +enters the payload-free result row. Unavailable or incomplete matching is diagnosed and +cannot pass a read verifier. + +Two count-oriented tasks have an equally narrow alternative evidence shape. R2 accepts an +exact `total_count` only when the request arguments contain the seeded project ID; R6 accepts +only grouped counts containing both seeded project IDs with their exact API-confirmed counts. +No other read task receives this relaxation, and both transports still persist labels only. Provider usage is a different measurement: where the driver supplies it, the harness keeps input, output, cache-read, and cache-creation usage. Tool-result sizing describes one source @@ -204,7 +219,7 @@ For each task repetition, the runner creates a fresh project and only the fixtur declared by that task. The live sequence is: ```text -seed -> drive -> verify -> teardown -> append row +seed -> drive -> verify -> capture non-secret seed shape -> teardown -> append row ``` The row is assembled as the task progresses; teardown runs in `finally` before that row is @@ -213,10 +228,18 @@ stdio server is launched for each driven task. The server environment is built f `HOME`, the three Plane connection values, and explicit `--server-env` additions; unrelated parent environment variables are not inherited. +Result rows retain only seeded entity kinds and randomization namespaces. They never contain +target entity IDs or randomized truth values. Each repetition has an independent persisted +`fixture_seed_id`; its truth is reproducible from that seed plus namespace without exposing +any later repetition's independent sentinel. + The first line of a new result file is a meta row containing the run identity, label, server, -battery, requested model/tier, resolved model, driver, provider, and Git SHA. Resume checks those identities, -skips completed task/repetition keys, and reruns rows that contain recorded errors. Result -rows preserve the common fields consumed by `evals.report` and existing JSONL readers. +battery, requested model/tier, resolved model, driver, provider, Git SHA, exact task-id list, +and repetition count. Reports compare raw `(task_id, rep)` histories with that exact set +before latest-wins deduplication. Resume checks run identity and only appends replacements. +A repeated key is valid only when every occurrence except the authoritative last row is +retryable; prior terminal rows remain genuine duplicates and make the run incomplete. +Result rows preserve the common fields consumed by `evals.report` and existing JSONL readers. ## Module layout diff --git a/evals/README.md b/evals/README.md index 56d97bc4..1fa92200 100644 --- a/evals/README.md +++ b/evals/README.md @@ -114,9 +114,12 @@ habit; use it only when the more sensitive, larger sidecar is justified. Read-task provenance is stricter than “a call happened.” Seeders place a hidden per-run sentinel on the target entity, and the API driver or CLI proxy records only whether a -successful response exposed it. Result rows contain the matched sentinel label, never the -sentinel value or response body. Thus an unrelated successful call cannot satisfy provenance; -a driver path where response matching is unavailable is diagnosed and fails closed. +successful response exposed it **and its request targeted that seeded entity ID**. CLI +proxies receive only target IDs and one-way value fingerprints through a private one-shot +file; the raw sentinel is absent even if that file is inspected. The proxy unlinks it before +Plane MCP starts. Result rows contain the matched label, never the sentinel value or response body. Thus an unrelated +successful response cannot satisfy provenance; unavailable or incomplete matching is +diagnosed and fails closed. ### Reading results @@ -150,11 +153,17 @@ independent sampling units and assumes comparable task instances under both labe printed paired task count—and the resulting wide interval for small samples—matters. Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, -prompts, and catalog revision. It contains exactly what the agent is asked and no expectation -about how the answer should be produced. A table that mixes fingerprints normally compares -different questions, even when task IDs are the same, so `evals.report --table` warns when its -input rows span fingerprints. A revision can document a deliberately comparable structural -change when prompts and verifiers remain unchanged. +prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and +fixture names. The battery contains exactly what the agent is asked and no expectation about +how the answer should be produced. All report paths refuse with exit 2 before printing +measurements when persisted battery identities differ, including mixed rows within one file. + +Canonical model, provider, driver, and server differences are comparison treatments. Declare +each intentional difference with `--vary`, for example `--vary resolved_model` or +`--vary provider,resolved_model`; any undeclared difference also refuses. Battery cannot be +declared as varying. Requested tier/model names remain provenance rather than canonical +identity, and the provider-reported realized model is printed as evidence instead of being +mechanically equated with the configured model. The hash excludes fixtures and verifier bodies. `CATALOG_REVISION` in `tasks/catalog.py` closes that gap: bump it whenever an excluded change redefines what a task asks, and explain @@ -277,9 +286,17 @@ Keep such scripts outside version control — `localdev/` is ignored for exactly - Run completeness uses an explicit skip taxonomy: known capabilities the environment does not provide (an allowlisted `env:plan-gated:` or the exact reason `env:no-activity-worker`) are expected skips. + The task/capability pair must also match the task's declared fixture needs: for example, + `env:plan-gated:customers` is expected for L4 but unexpected for W1 or C1. They reduce **EXECUTION COVERAGE** but do not break **RUN COMPLETE**. A dirty environment (`env:fixture-collision:*`) and every unrecognised reason are unexpected and make the run incomplete; there is intentionally no catch-all for new `env:*` reasons. +- New result headers declare the exact task-id subset and repetition count. Completeness + compares raw `(task_id, rep)` occurrences with that declaration before latest-wins + deduplication, naming missing and unexpected keys (including duplicate excess). +- Report headlines use a task-cluster bootstrap interval; the pooled repetition rate and + Wilson interval remain visible but are explicitly labeled as pooled. A/B and surface-table + reports warn when any input lacks a tool-manifest fingerprint. - A feature switched **off for a project** is not a plan gate — it is configuration the harness sets itself, and W11 exists to measure what an agent does when it meets one. - **Gated endpoints returning 402 on a workspace that should work.** Feature flags are diff --git a/evals/result_lifecycle.py b/evals/result_lifecycle.py new file mode 100644 index 00000000..ae072028 --- /dev/null +++ b/evals/result_lifecycle.py @@ -0,0 +1,24 @@ +"""Shared terminal/retryable classification for persisted evaluation results.""" + +from __future__ import annotations + +from typing import Any + +from evals.results import TaskResult +from evals.skip_taxonomy import is_expected_environment_capability_skip + + +def is_terminal_result(row: TaskResult | dict[str, Any]) -> bool: + """Return whether a result is authoritative rather than eligible for retry.""" + result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) + error_class = result.error_class + if isinstance(error_class, str) and error_class.startswith("infra_"): + return False + if result.error is not None or result.cleanup_error is not None: + return False + if result.skipped is not None: + return is_expected_environment_capability_skip(result.skipped, task_id=result.task_id) + return True + + +__all__ = ["is_terminal_result"] diff --git a/evals/results.py b/evals/results.py index c9c1432f..4212d831 100644 --- a/evals/results.py +++ b/evals/results.py @@ -13,7 +13,9 @@ ) from evals.tool_names import split_plane_and_client_calls -RESULT_SCHEMA_VERSION = 3 +RESULT_SCHEMA_VERSION = 6 + +TraceIntegrityReason = Literal["recorder_loss", "protocol_violation", "result_pair_mismatch"] # ``apply_agent_result`` owns this explicit partition. A reflection test compares # it with every TaskResult dataclass field so additions cannot disappear silently. @@ -23,6 +25,9 @@ "provider_stop_reason", "hit_max_iterations", "result_pair_mismatch", + "trace_integrity", + "trace_integrity_reason", + "tool_manifest_fingerprint", "token_count_failures", "result_tokens_estimated", "calls", @@ -51,9 +56,11 @@ "schema_version", "row_type", "run_id", + "fixture_seed_id", "ts", "git_sha", "battery", + "task_fingerprint", "label", "driver", "server", @@ -69,6 +76,8 @@ "error", "error_class", "cleanup_error", + "seeded_entity_kinds", + "randomized_seed_namespaces", ) @@ -126,6 +135,9 @@ class AgentRun: usage_per_iteration: list[Usage] = field(default_factory=list) cum_input_tokens: int | None = None result_pair_mismatch: bool = False + trace_integrity: bool = True + trace_integrity_reason: TraceIntegrityReason | None = None + tool_manifest_fingerprint: str | None = None token_count_failures: int = 0 # False means a tokenizer/backend counter was used for every result; True # means at least one result used the shared character estimate. None lets @@ -148,15 +160,22 @@ class TaskResult: field added since. Version 1 defines wall_time_s as CLI invocation time only — earlier Claude/Antigravity/OpenCode rows also include a few ms of harness setup. Version 2 adds run-completeness metadata and cleanup failure recording. Version 3 records only - response-evidence labels (never Plane response bodies) plus trace availability. + response-evidence labels (never Plane response bodies) plus trace availability. Version + 4 adds the task-local question fingerprint used by future intersection comparisons. + Version 5 adds typed trace integrity and the observed tool-manifest fingerprint. + Version 6 adds the reproducible per-repetition fixture seed id, non-secret fixture kinds, + and randomization namespaces. Target entity ids and randomized truth values are + deliberately excluded. """ schema_version: int = RESULT_SCHEMA_VERSION row_type: str | None = None run_id: str = "" + fixture_seed_id: str = "" ts: str = "" git_sha: str = "" battery: str = "" + task_fingerprint: str = "" label: str = "" driver: str = "" provider: str | None = None @@ -175,11 +194,16 @@ class TaskResult: error: str | None = None error_class: str | None = None cleanup_error: str | None = None + seeded_entity_kinds: list[str] = field(default_factory=list) + randomized_seed_namespaces: list[str] = field(default_factory=list) final_text: str = "" stop_reason: str | None = None provider_stop_reason: str | None = None hit_max_iterations: bool = False result_pair_mismatch: bool = False + trace_integrity: bool = True + trace_integrity_reason: TraceIntegrityReason | None = None + tool_manifest_fingerprint: str | None = None token_count_failures: int = 0 result_tokens_estimated: bool | None = None calls: list[CallRecord] = field(default_factory=list) @@ -262,9 +286,11 @@ def usage_row(item: Usage) -> dict[str, int]: row: dict[str, Any] = { "schema_version": self.schema_version, "run_id": self.run_id, + "fixture_seed_id": self.fixture_seed_id, "ts": self.ts, "git_sha": self.git_sha, "battery": self.battery, + "task_fingerprint": self.task_fingerprint, "label": self.label, "driver": self.driver, "provider": self.provider, @@ -283,11 +309,16 @@ def usage_row(item: Usage) -> dict[str, int]: "error": self.error, "error_class": self.error_class, "cleanup_error": self.cleanup_error, + "seeded_entity_kinds": list(self.seeded_entity_kinds), + "randomized_seed_namespaces": list(self.randomized_seed_namespaces), "final_text": self.final_text, "stop_reason": self.stop_reason, "provider_stop_reason": self.provider_stop_reason, "hit_max_iterations": self.hit_max_iterations, "result_pair_mismatch": self.result_pair_mismatch, + "trace_integrity": self.trace_integrity, + "trace_integrity_reason": self.trace_integrity_reason, + "tool_manifest_fingerprint": self.tool_manifest_fingerprint, "token_count_failures": self.token_count_failures, "result_tokens_estimated": self.result_tokens_estimated, "calls": calls, @@ -385,9 +416,11 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: schema_version=int(row.get("schema_version") or 0), row_type=(str(row["row_type"]) if row.get("row_type") is not None else None), run_id=str(row.get("run_id") or ""), + fixture_seed_id=str(row.get("fixture_seed_id") or ""), ts=str(row.get("ts") or ""), git_sha=str(row.get("git_sha") or ""), battery=str(row.get("battery") or ""), + task_fingerprint=str(row.get("task_fingerprint") or ""), label=str(row.get("label") or ""), driver=str(row.get("driver") or ""), provider=(str(row["provider"]) if row.get("provider") is not None else None), @@ -406,6 +439,16 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: error=(str(row["error"]) if row.get("error") is not None else None), error_class=(str(row["error_class"]) if row.get("error_class") is not None else None), cleanup_error=(str(row["cleanup_error"]) if row.get("cleanup_error") is not None else None), + seeded_entity_kinds=( + [str(kind) for kind in row["seeded_entity_kinds"]] + if isinstance(row.get("seeded_entity_kinds"), list) + else [] + ), + randomized_seed_namespaces=( + [str(namespace) for namespace in row["randomized_seed_namespaces"]] + if isinstance(row.get("randomized_seed_namespaces"), list) + else [] + ), final_text=str(row.get("final_text") or ""), stop_reason=(str(row["stop_reason"]) if row.get("stop_reason") is not None else None), provider_stop_reason=( @@ -413,6 +456,20 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ), hit_max_iterations=bool(row.get("hit_max_iterations")), result_pair_mismatch=bool(row.get("result_pair_mismatch")), + trace_integrity=bool(row.get("trace_integrity", True)), + trace_integrity_reason=( + str(row["trace_integrity_reason"]) + if row.get("trace_integrity_reason") + in { + "recorder_loss", + "protocol_violation", + "result_pair_mismatch", + } + else None + ), + tool_manifest_fingerprint=( + str(row["tool_manifest_fingerprint"]) if row.get("tool_manifest_fingerprint") is not None else None + ), token_count_failures=int(row.get("token_count_failures") or 0), result_tokens_estimated=( bool(row["result_tokens_estimated"]) if row.get("result_tokens_estimated") is not None else None @@ -611,6 +668,9 @@ def agent_run_to_task_result( provider_stop_reason=run.provider_stop_reason, hit_max_iterations=hit_max, result_pair_mismatch=run.result_pair_mismatch, + trace_integrity=run.trace_integrity, + trace_integrity_reason=run.trace_integrity_reason, + tool_manifest_fingerprint=run.tool_manifest_fingerprint, token_count_failures=run.token_count_failures + local_token_count_failures, result_tokens_estimated=result_tokens_estimated, result_tokens_mode=result_tokens_mode, @@ -643,6 +703,7 @@ def agent_run_to_harness_dict( "AgentRun", "CallRecord", "TaskResult", + "TraceIntegrityReason", "Usage", "agent_run_to_harness_dict", "agent_run_to_task_result", diff --git a/evals/runner/live.py b/evals/runner/live.py index 0a4ddc76..7a38406f 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -15,11 +15,12 @@ from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS from evals.evidence import configured_evidence_labels -from evals.report.load import load_rows +from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys from evals.report.summary import completeness_statement, execution_coverage_statement, summarize from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown -from evals.tasks.catalog import battery_fingerprint, task_author +from evals.seed.identities import capture_seed_artifacts +from evals.tasks.catalog import battery_fingerprint, task_author, task_fingerprint from evals.tasks.prompts import PromptBindError, format_task_prompt from evals.tasks.skip import TaskSkipped @@ -117,6 +118,7 @@ async def run_agent_task_via_driver( cwd=Path(__file__).resolve().parent.parent.parent, evidence_sentinels=ctx.get("evidence_sentinels"), evidence_targets=ctx.get("evidence_targets"), + evidence_aggregates=ctx.get("evidence_aggregates"), ) return agent_run_to_task_result(agent_run) @@ -139,9 +141,11 @@ def _make_task_row( ) -> TaskResult: return TaskResult( run_id=run_id, + fixture_seed_id=uuid.uuid4().hex, ts=datetime.now(timezone.utc).isoformat(), git_sha=git_revision, battery=battery, + task_fingerprint=task_fingerprint(task), label=label, driver=driver_name, provider=provider, @@ -171,7 +175,7 @@ def _seed_fixtures( try: seed( plane, - run_id=uuid.uuid4().hex, + run_id=row.fixture_seed_id, needs=task_needs, ctx=context, task_id=str(task["id"]), @@ -248,6 +252,10 @@ async def _drive_agent( ) return None except Exception as exc: + if hasattr(exc, "trace_integrity"): + row.trace_integrity = bool(exc.trace_integrity) + row.trace_integrity_reason = getattr(exc, "trace_integrity_reason", None) + row.tool_manifest_fingerprint = getattr(exc, "tool_manifest_fingerprint", None) if is_api_driver: agent_error_class = "infra_api" else: @@ -311,6 +319,36 @@ def _record_cli_infra_stop( return True +def _record_trace_infra( + row: TaskResult, + agent: TaskResult, + *, + task: dict[str, Any], + repetition: int, +) -> bool: + """Make recorder/protocol trace loss completeness-visible infrastructure.""" + if agent.trace_integrity or agent.trace_integrity_reason == "result_pair_mismatch": + return False + error_class = "infra_protocol" if agent.trace_integrity_reason == "protocol_violation" else "infra_trace" + detail = next( + ( + note + for note in agent.driver_notes + if isinstance(note, str) and note.startswith(("proxy_sidecar_incomplete", "proxy_sidecar_empty")) + ), + f"trace_integrity={agent.trace_integrity_reason or 'recorder_loss'}", + ) + row.success = False + row.error_class = error_class + row.error = detail + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{error_class}]: {detail}", + file=sys.stderr, + ) + return True + + async def _verify_task( plane: Any, task: dict[str, Any], @@ -334,6 +372,8 @@ async def _verify_task( "evidence_trace_available": agent.evidence_trace_available, "driver_notes": list(agent.driver_notes), "result_pair_mismatch": agent.result_pair_mismatch, + "trace_integrity": agent.trace_integrity, + "trace_integrity_reason": agent.trace_integrity_reason, }, ) row.success = bool(ok) @@ -448,18 +488,31 @@ async def _run_task_repetition( requested_tier=requested_tier, model_id=model_id, ) - if not _record_cli_infra_stop( + infra_stop = _record_cli_infra_stop( row, agent, task=task, repetition=repetition, driver_name=driver_name, - ): + ) + trace_infra = False + if not infra_stop: + trace_infra = _record_trace_infra( + row, + agent, + task=task, + repetition=repetition, + ) + if not infra_stop and not trace_infra: await _verify_task(plane, task, context, agent, row=row, repetition=repetition) except Exception as exc: # Anything outside seed/driver/verify wraps. _record_unexpected(row, exc, task=task, repetition=repetition, context=context) finally: + try: + row.seeded_entity_kinds, row.randomized_seed_namespaces = capture_seed_artifacts(context) + except Exception as exc: + _record_unexpected(row, exc, task=task, repetition=repetition, context=context) _remove_fixtures(plane, context, row) return row @@ -531,6 +584,8 @@ async def run_live( provider=provider_id, git_sha=git_revision, expected_rows=total_runs, + expected_task_ids=[str(task["id"]) for task in tasks], + expected_reps=reps, ) if maybe_write_run_meta(out_path, meta): print(f"wrote meta header battery={battery} label={label}", flush=True) @@ -618,15 +673,20 @@ async def run_live( flush=True, ) selected_task_ids = {str(task["id"]) for task in tasks} + raw_result_rows = load_rows(out_path, dedupe="none") + run_keys = validate_run_keys( + raw_result_rows, + RunExpectation(tuple(str(task["id"]) for task in tasks), reps, label), + ) result_rows = [ row - for row in load_rows(out_path) + for row in dedupe_rows_latest(raw_result_rows) if row.label == label and (not row.battery or row.battery == battery) and row.task_id in selected_task_ids and 0 <= row.rep < reps ] - summary = summarize(result_rows, expected_rows=total_runs) + summary = summarize(result_rows, expected_rows=total_runs, run_keys=run_keys) if summary.aggregate_n: rate = summary.aggregate_k / summary.aggregate_n print(f"success: {summary.aggregate_k}/{summary.aggregate_n} ({rate:.1%})", flush=True) diff --git a/evals/runner/meta.py b/evals/runner/meta.py index 9cd02855..089d81e5 100644 --- a/evals/runner/meta.py +++ b/evals/runner/meta.py @@ -47,9 +47,22 @@ def make_run_meta_row( requested_tier: str | None = None, resolved_model: str | None = None, expected_rows: int | None = None, + expected_task_ids: list[str] | tuple[str, ...] | None = None, + expected_reps: int | None = None, ts: str | None = None, ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" + if (expected_task_ids is None) != (expected_reps is None): + raise ValueError("expected_task_ids and expected_reps must be declared together") + if expected_task_ids is not None and expected_reps is not None: + task_ids = [str(task_id) for task_id in expected_task_ids] + if not task_ids or any(not task_id for task_id in task_ids) or len(set(task_ids)) != len(task_ids): + raise ValueError("expected_task_ids must contain unique non-empty ids") + if expected_reps < 1: + raise ValueError("expected_reps must be positive") + exact_rows = len(task_ids) * expected_reps + if expected_rows is not None and expected_rows != exact_rows: + raise ValueError(f"expected_rows={expected_rows} disagrees with exact expectation={exact_rows}") row = { "schema_version": RESULT_SCHEMA_VERSION, "row_type": "meta", @@ -68,6 +81,10 @@ def make_run_meta_row( } if expected_rows is not None: row["expected_rows"] = expected_rows + if expected_task_ids is not None: + row["expected_task_ids"] = task_ids + if expected_reps is not None: + row["expected_reps"] = expected_reps return row diff --git a/evals/runner/resume.py b/evals/runner/resume.py index 044a793d..368a5d71 100644 --- a/evals/runner/resume.py +++ b/evals/runner/resume.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any +from evals.result_lifecycle import is_terminal_result from evals.results import TaskResult -from evals.skip_taxonomy import is_expected_environment_capability_skip from .meta import is_meta_or_non_task_row @@ -21,17 +21,7 @@ def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: add them; fixture collisions and unknown skips may be repairable. Pure function — unit-tested without the live battery. """ - result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) - error_class = result.error_class - if isinstance(error_class, str) and error_class.startswith("infra_"): - return False - if result.error is not None: - return False - if result.cleanup_error is not None: - return False - if result.skipped is not None: - return is_expected_environment_capability_skip(result.skipped) - return True + return is_terminal_result(row) def _resume_field_mismatch( diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 85357ddb..4bad73cf 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -28,6 +28,7 @@ ) from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.token_counting import estimate_result_tokens +from evals.tool_manifest import tool_manifest_fingerprint from tests.evals.conftest import case_params @@ -56,15 +57,20 @@ def add_tool_results(self, results: list[ToolResult]) -> None: class FakeMcpSession: - def __init__(self, results: list[Any] | None = None) -> None: + def __init__(self, results: list[Any] | None = None, tool_pages: list[Any] | None = None) -> None: self.results = deque(results or []) + self.tool_pages = deque(tool_pages or []) self.initialized = False self.called: list[tuple[str, dict[str, Any]]] = [] + self.list_cursors: list[str | None] = [] async def initialize(self) -> None: self.initialized = True - async def list_tools(self) -> Any: + async def list_tools(self, cursor: str | None = None) -> Any: + self.list_cursors.append(cursor) + if self.tool_pages: + return self.tool_pages.popleft() return SimpleNamespace( tools=[ SimpleNamespace( @@ -99,7 +105,14 @@ async def session_factory(_params): ) -def run_driver(driver: ApiDriver, *, max_turns: int = 5, evidence_sentinels=None): +def run_driver( + driver: ApiDriver, + *, + max_turns: int = 5, + evidence_sentinels=None, + evidence_targets=None, + evidence_aggregates=None, +): return driver.run_task( "do it", {"SAFE": "1"}, @@ -107,6 +120,8 @@ def run_driver(driver: ApiDriver, *, max_turns: int = 5, evidence_sentinels=None max_turns, system="system", evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, ) @@ -314,9 +329,64 @@ def _api_driver_flags_result_id_mismatch(): run = run_driver(make_driver(backend, session)) assert run.result_pair_mismatch is True + assert run.trace_integrity is False + assert run.trace_integrity_reason == "result_pair_mismatch" assert [call["result_chars"] for call in run.calls] == [0, 4] +def test_api_driver_aggregates_every_tools_list_page_before_fingerprinting(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + tools = [ + {"name": "alpha", "inputSchema": {"type": "object"}}, + {"name": "beta", "inputSchema": {"type": "object"}}, + ] + session = FakeMcpSession( + tool_pages=[ + {"tools": [tools[0]], "nextCursor": "page-2"}, + {"tools": [tools[1]]}, + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert session.list_cursors == [None, "page-2"] + assert run.tool_manifest_fingerprint == tool_manifest_fingerprint(tools) + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["alpha", "beta"] + + +def test_api_driver_invalidates_manifest_after_tools_list_changed(monkeypatch): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + session = FakeMcpSession() + + @asynccontextmanager + async def fake_stdio_client(_params): + yield object(), object() + + class FakeClientSessionContext: + def __init__(self, _read, _write, *, message_handler): + self.message_handler = message_handler + + async def __aenter__(self): + return session + + async def __aexit__(self, *_args): + await self.message_handler(SimpleNamespace(root=SimpleNamespace(method="notifications/tools/list_changed"))) + return None + + monkeypatch.setattr("evals.drivers.driver.stdio_client", fake_stdio_client) + monkeypatch.setattr("evals.drivers.driver.ClientSession", FakeClientSessionContext) + driver = ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + server_command=["fake-server"], + ) + + run = run_driver(driver) + + assert run.tool_manifest_fingerprint is None + + def _api_driver_iteration_cap_only_flags_mid_tool_loop(): backend = FakeBackend( [ @@ -396,7 +466,9 @@ def _api_driver_records_only_matching_evidence_labels(): ) session = FakeMcpSession( [ - ToolResult(call_id="a", text="ordinary workspace response"), + # A real response carrying the sentinel is insufficient when the request + # targeted an unrelated entity (write-there/read-back bypass). + ToolResult(call_id="a", text=f"state={sentinel}"), ToolResult(call_id="b", text=f"state={sentinel}"), ] ) @@ -404,6 +476,7 @@ def _api_driver_records_only_matching_evidence_labels(): run = run_driver( make_driver(backend, session), evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target"]}, ) assert run.evidence_trace_available is True @@ -412,6 +485,78 @@ def _api_driver_records_only_matching_evidence_labels(): assert "result_text" not in run.calls[1] +def test_api_driver_records_only_exact_target_bound_aggregate_evidence(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "count_work_items", {"pql": 'project = "project-other"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("b", "count_work_items", {"pql": 'project = "project-1"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("c", "count_work_items", {"pql": 'project = "project-1"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("d", "count_work_items", {"group_by": "project_id"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("e", "count_work_items", {"group_by": "project_id"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="a", text='{"total_count": 4}'), + ToolResult(call_id="b", text='{"total_count": 3}'), + ToolResult(call_id="c", text='{"total_count": 4}'), + ToolResult( + call_id="d", + text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 4}}}'), + ), + ToolResult( + call_id="e", + text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 5}}}'), + ), + ] + ) + + run = run_driver( + make_driver(backend, session), + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1", "project-2"]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": 4}, + {"kind": "grouped_counts", "values": {"project-1": 2, "project-2": 5}}, + ] + }, + max_turns=6, + ) + + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[1]["observed_sentinels"] == [] + assert run.calls[2]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.calls[3]["observed_sentinels"] == [] + assert run.calls[4]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + + _API_DRIVER_CASES = case_params( _api_driver_multi_turn_tool_loop_and_usage_accumulation, _api_driver_refusal_records_calls_but_executes_nothing, diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 24c234c4..1077793c 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -26,7 +26,7 @@ run_cli_subprocess, wait_for_proxy_meta, ) -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput +from evals.drivers.driver import CliDriver, CliLaunch, CliOutput, CliOutputError from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from tests.evals.conftest import case_params @@ -333,7 +333,13 @@ def write_complete_sidecar(path: Path, tool: str) -> None: "duration_ms": 1, "seq": 1, }, - {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}, + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + }, ] path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") @@ -390,7 +396,15 @@ def test_old_payload_free_sidecar_still_parses(tmp_path: Path): } ) + "\n" - + json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}) + + json.dumps( + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + ) + "\n", encoding="utf-8", ) @@ -401,6 +415,52 @@ def test_old_payload_free_sidecar_still_parses(tmp_path: Path): assert "result_text" not in calls[0] +def test_cli_parse_failure_retains_lossy_sidecar_integrity(tmp_path: Path): + class BrokenOutputDriver(CliDriver): + name = "broken-output-cli" + + def write_mcp_config(self, temp_dir, *, task_cwd, server_command, child_env): + del temp_dir, child_env + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command(self, prompt, *, model, max_turns, system, launch): + del prompt, model, max_turns, system, launch + return ["broken-output"] + + def parse_output(self, proc, *, task_cwd, max_turns, notes): + del proc, task_cwd, max_turns, notes + raise CliOutputError("cannot parse output") + + driver: BrokenOutputDriver + + def fake_run(command, **kwargs): + del kwargs + rows = [ + { + "row_type": "proxy_meta", + "unmatched_responses": 1, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 0, + "tool_request_count": 0, + } + ] + driver.sidecar_path.write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="bad", stderr="") + + driver = BrokenOutputDriver(runner=fake_run, use_proxy=True) + + with pytest.raises(RuntimeError, match="cannot parse output") as exc_info: + driver.run_task("go", {}, None, 1, cwd=tmp_path) + + assert exc_info.value.trace_integrity is False + assert exc_info.value.trace_integrity_reason == "recorder_loss" + + def _apply_proxy_sidecar_replaces_when_nonempty(tmp_path): side = tmp_path / "s.jsonl" side.write_text( @@ -484,7 +544,15 @@ def _apply_proxy_with_skipped_row_defers_to_richer_cli(tmp_path): } ), "{corrupted mid-stream row", - json.dumps({"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False}), + json.dumps( + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + ), ] p.write_text("\n".join(rows) + "\n", encoding="utf-8") cli = [ @@ -546,6 +614,9 @@ def _load_proxy_sidecar_sorts_by_seq(tmp_path): "unmatched_responses": 0, "notifications": 0, "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 1, + "tool_request_count": 1, "child_killed": False, }, ] @@ -685,6 +756,96 @@ def fake_run(cmd, **kwargs): assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") +@pytest.mark.parametrize( + ("Driver", "bin_key"), + [ + pytest.param(ClaudeCliDriver, "claude_bin", id="claude-config"), + pytest.param(CodexCliDriver, "codex_bin", id="codex-argv"), + pytest.param(AntigravityCliDriver, "agy_bin", id="antigravity-home-config"), + pytest.param(OpencodeCliDriver, "opencode_bin", id="opencode-cwd-config"), + ], +) +def test_cli_agent_surfaces_never_contain_evidence_sentinel( + tmp_path: Path, + Driver: type[CliDriver], + bin_key: str, +): + sentinel = "hidden-target-fact-7b0a1f9c" + seen: dict[str, object] = {} + + def fake_run(cmd, **kwargs): + configs: list[dict] = [] + proxy_args: list[str] | None = None + if Driver is ClaudeCliDriver: + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + configs.append(json.loads(config_path.read_text())) + proxy_args = configs[0]["mcpServers"]["plane"]["args"] + elif Driver is CodexCliDriver: + for value in cmd: + prefix = "mcp_servers.plane.args=" + if value.startswith(prefix): + proxy_args = json.loads(value[len(prefix) :]) + elif Driver is AntigravityCliDriver: + fake_home = Path(kwargs["env"]["HOME"]) + for rel in ( + Path(".gemini/config/mcp_config.json"), + Path(".gemini/antigravity-cli/mcp_config.json"), + ): + configs.append(json.loads((fake_home / rel).read_text())) + proxy_args = configs[0]["mcpServers"]["plane"]["args"] + else: + config_path = Path(kwargs["cwd"]) / "opencode.json" + configs.append(json.loads(config_path.read_text())) + proxy_args = configs[0]["mcp"]["plane"]["command"] + + assert proxy_args is not None and "--evidence-file" in proxy_args + evidence_path = Path(proxy_args[proxy_args.index("--evidence-file") + 1]) + launch_cwd = Path(kwargs["cwd"]).resolve() + assert not evidence_path.resolve().is_relative_to(launch_cwd) + # Even a lazy MCP launcher leaves only non-invertible fingerprints for a + # shell-capable agent that follows the pathname before proxy startup. + evidence_config = evidence_path.read_text(encoding="utf-8") + seen["surface"] = json.dumps({"argv": cmd, "configs": configs, "evidence_file": evidence_config}) + out = ( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + } + ) + if Driver is ClaudeCliDriver + else "{}" + ) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + kwargs = { + "runner": fake_run, + "python_bin": sys.executable, + "use_proxy": True, + bin_key: "fake-bin", + } + if Driver is CodexCliDriver: + kwargs["allow_live"] = True + driver = Driver(**kwargs) + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="test-model", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + + surface = str(seen["surface"]) + assert sentinel not in surface + assert EVIDENCE_SENTINELS_ENV not in surface + + def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") @@ -727,6 +888,9 @@ def _timeout_harvests_sidecar_calls(tmp_path): "unmatched_responses": 0, "notifications": 0, "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 1, + "tool_request_count": 1, "child_killed": False, "evidence_trace_available": True, }, @@ -738,8 +902,9 @@ def fake_run(cmd, **kwargs): # Sidecar path is in the same temp dir as mcp.json for Claude. # Find sidecar from proxy args in mcp config. mcp = json.loads(cfg.read_text()) - assert EVIDENCE_SENTINELS_ENV in mcp["mcpServers"]["plane"]["env"] + assert EVIDENCE_SENTINELS_ENV not in mcp["mcpServers"]["plane"]["env"] args = mcp["mcpServers"]["plane"]["args"] + assert "--evidence-file" in args log_idx = args.index("--log") + 1 side = Path(args[log_idx]) side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") @@ -753,6 +918,7 @@ def fake_run(cmd, **kwargs): max_turns=1, cwd=tmp_path, evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, ) assert run.stopped_reason == "timeout" assert run.call_source == "proxy" @@ -842,6 +1008,8 @@ def fake_run(cmd, **kwargs): "row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, "evidence_trace_available": True, } side.write_text( @@ -858,6 +1026,7 @@ def fake_run(cmd, **kwargs): max_turns=1, cwd=tmp_path, evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, ) assert run.call_source == "proxy" diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 2c34782b..5c388dbb 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -26,6 +26,9 @@ ) from evals.runner import live as runner_live from evals.runner.live import stdio_server_env +from evals.seed.identities import record_seeded_entity +from evals.seed.randomize import random_truth_token, record_randomized_truth +from evals.tasks.catalog import task_fingerprint from evals.tasks.skip import TaskSkipped from tests.evals.conftest import _data_rows, case_params @@ -125,6 +128,7 @@ def boom_seed(plane, run_id, needs, ctx, task_id=None): assert "HttpError" in (row["error"] or "") assert "identifier" in (row["error"] or "").lower() assert row["battery"] # fingerprint written + assert row["task_fingerprint"] == task_fingerprint(task) assert row["requested_model"] == "standard" assert row["requested_tier"] == "standard" assert row["resolved_model"] == "sonnet" @@ -157,7 +161,7 @@ def seed_without_bug_type(plane, run_id, needs, ctx, task_id=None): monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) task = _taxonomy_task( - "BUGTYPE", + "S1", lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), needs={"bug_type"}, ) @@ -265,7 +269,11 @@ class BoomDriver: name = "claude-cli" def run_task(self, *args, **kwargs): - raise RuntimeError("claude cli failed: json_parse_failed") + error = RuntimeError("claude cli failed: json_parse_failed") + error.trace_integrity = False + error.trace_integrity_reason = "recorder_loss" + error.tool_manifest_fingerprint = None + raise error monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) @@ -291,6 +299,8 @@ def run_task(self, *args, **kwargs): row = _data_rows(out)[0] assert row["error_class"] == "infra_cli" assert "RuntimeError" in (row["error"] or "") + assert row["trace_integrity"] is False + assert row["trace_integrity_reason"] == "recorder_loss" def _run_timeout_agent_is_infra_cli(tmp_path, monkeypatch, _capsys): @@ -370,7 +380,11 @@ def _run_error_during_execution_is_infra_cli(tmp_path, monkeypatch, _capsys): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr( + runner_live, + "get_driver", + lambda name, **kw: ClaudeCliDriver(runner=fake_run, use_proxy=False), + ) verify_calls: list[Any] = [] @@ -423,7 +437,11 @@ def _run_error_max_turns_is_task_path(tmp_path, monkeypatch, _capsys): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") - monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: ClaudeCliDriver(runner=fake_run)) + monkeypatch.setattr( + runner_live, + "get_driver", + lambda name, **kw: ClaudeCliDriver(runner=fake_run, use_proxy=False), + ) verify_calls: list[Any] = [] @@ -625,7 +643,7 @@ async def verify_ok(plane, ctx, run): assert row["server"] == "local" -def _run_multi_rep_uses_fresh_seed_and_teardown_per_rep(tmp_path, monkeypatch, _capsys): +def _run_multi_rep_uses_fresh_fixture_seed_and_teardown_per_rep(tmp_path, monkeypatch, _capsys): out = tmp_path / "multi.jsonl" fake_plane = MagicMock() monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) @@ -674,6 +692,9 @@ async def verify_ok(plane, ctx, run): assert len(set(seed_ids)) == 3 assert teardown_projects == seed_ids rows = _data_rows(out) + assert {row["fixture_seed_id"] for row in rows} == set(seed_ids) + assert len({row["run_id"] for row in rows}) == 1 + rows = _data_rows(out) assert [row["rep"] for row in rows] == [0, 1, 2] assert all(row["success"] is True for row in rows) @@ -708,6 +729,7 @@ def run_task(self, *args, **kwargs): context = { "project_name": "EVAL deadbeef", "evidence_sentinels": {TARGET_ENTITY_EVIDENCE: [sentinel]}, + "evidence_targets": {TARGET_ENTITY_EVIDENCE: ["project-1"]}, } row = asyncio.run( @@ -721,6 +743,7 @@ def run_task(self, *args, **kwargs): ) assert captured["evidence_sentinels"] == context["evidence_sentinels"] + assert captured["evidence_targets"] == context["evidence_targets"] assert row.evidence_trace_available is True assert row.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] assert sentinel not in json.dumps(row.to_row()) @@ -822,7 +845,7 @@ async def fake_drive(**kwargs): _run_verifier_exception_is_task_error, _run_external_server_records_observed_calls, _run_success_keeps_requested_and_resolved_models, - _run_multi_rep_uses_fresh_seed_and_teardown_per_rep, + _run_multi_rep_uses_fresh_fixture_seed_and_teardown_per_rep, _run_passes_server_cmd_to_non_claude, _run_reports_progress_per_repetition, ) @@ -968,16 +991,18 @@ def seed_l2(plane, run_id, needs, ctx, task_id=None): @pytest.mark.parametrize( - ("reason", "expected_rc", "verdict"), + ("reason", "task_id", "needs", "expected_rc", "verdict"), [ - ("env:plan-gated:customers", 0, "RUN COMPLETE:"), - ("env:no-activity-worker", 0, "RUN COMPLETE:"), - ("env:plan-gated:customerz", 1, "RUN INCOMPLETE:"), - ("env:fixture-collision:customers:Acme Corp", 1, "RUN INCOMPLETE:"), - ("env:new-skip-reason", 1, "RUN INCOMPLETE:"), + ("env:plan-gated:customers", "L4", {"customer"}, 0, "RUN COMPLETE:"), + ("env:no-activity-worker", "L2", {"activity_feed"}, 0, "RUN COMPLETE:"), + ("env:plan-gated:customerz", "L4", {"customer"}, 1, "RUN INCOMPLETE:"), + ("env:fixture-collision:customers:Acme Corp", "C1", set(), 1, "RUN INCOMPLETE:"), + ("env:new-skip-reason", "R1", set(), 1, "RUN INCOMPLETE:"), ], ) -def test_run_live_completeness_skip_taxonomy(tmp_path, monkeypatch, capsys, reason, expected_rc, verdict): +def test_run_live_completeness_skip_taxonomy( + tmp_path, monkeypatch, capsys, reason, task_id, needs, expected_rc, verdict +): out = tmp_path / "out.jsonl" def skip_seed(*_args, **_kwargs): @@ -987,18 +1012,124 @@ def skip_seed(*_args, **_kwargs): monkeypatch.setattr(runner_live, "seed", skip_seed) monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) - tasks = [_taxonomy_task("R1", None), _taxonomy_task("R2", None)] + tasks = [_taxonomy_task(task_id, None, needs=needs)] rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) assert rc == expected_rc output = capsys.readouterr().out assert "success: 0/0" in output - assert "EXECUTION COVERAGE: 0/2 rows evaluated (0.0%)" in output - assert f"R1,R2 ({reason})" in output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in output + assert f"{task_id} ({reason})" in output assert verdict in output if reason.startswith("env:fixture-collision:"): - assert "unexpected skips=2 [fixture-collision=2]" in output + assert "unexpected skips=1 [fixture-collision=1]" in output + + +def test_persisted_seed_artifacts_never_contain_run_random_truth_or_entity_ids(tmp_path, monkeypatch): + out = tmp_path / "forensics.jsonl" + namespace = "C2.release" + seeded_values: dict[str, str] = {} + + def seed_forensics(_plane, run_id, needs, ctx, task_id=None): + del needs, task_id + ctx["run_id"] = run_id + sentinel = random_truth_token(ctx, namespace) + seeded_values.update(run_id=run_id, sentinel=sentinel) + ctx.update( + { + "project_name": "EVAL forensic", + "project_id": "project-1", + "evidence_sentinels": {TARGET_ENTITY_EVIDENCE: [sentinel]}, + "evidence_targets": {TARGET_ENTITY_EVIDENCE: ["item-1"]}, + } + ) + record_seeded_entity(ctx, "work_item", "item-1") + record_seeded_entity(ctx, "project", "project-1") + record_randomized_truth( + ctx, + namespace, + { + "intended_name": f"1.6.10-eval.{sentinel[:8]}", + "intended_changelog": f"ticket EVAL-{sentinel}", + }, + ) + + class InspectingDriver: + def run_task(self, *_args, **kwargs): + assert _data_rows(out) == [] + assert kwargs["evidence_sentinels"] == {TARGET_ENTITY_EVIDENCE: [seeded_values["sentinel"]]} + return AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", seed_forensics) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: InspectingDriver()) + task = { + "id": "R1", + "prompt": "do {project}", + "tags": {"read"}, + "needs": {"items"}, + "verify": verify_ok, + } + + assert asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) == 0 + row = _data_rows(out)[0] + assert row["fixture_seed_id"] == seeded_values["run_id"] + assert row["run_id"] != row["fixture_seed_id"] + assert row["seeded_entity_kinds"] == ["project", "work_item"] + assert row["randomized_seed_namespaces"] == [namespace] + persisted = json.dumps(row, sort_keys=True) + assert seeded_values["sentinel"] not in persisted + assert "project-1" not in persisted + assert "item-1" not in persisted + assert "seeded_entity_ids" not in row + assert "randomized_seed_choices" not in row + assert "evidence_sentinels" not in row + assert "evidence_targets" not in row + + +def test_same_run_repetitions_produce_different_random_truth_sentinels(tmp_path, monkeypatch): + out = tmp_path / "per-repetition-truth.jsonl" + namespace = "R7.states" + seeded: list[tuple[str, str]] = [] + + def seed_with_sentinel(_plane, run_id, needs, ctx, task_id=None): + del needs, task_id + ctx.update({"run_id": run_id, "project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + sentinel = random_truth_token(ctx, namespace) + seeded.append((run_id, sentinel)) + ctx["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: [sentinel]} + ctx["evidence_targets"] = {TARGET_ENTITY_EVIDENCE: [run_id]} + + class OkDriver: + def run_task(self, *_args, **_kwargs): + return AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", seed_with_sentinel) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: OkDriver()) + task = { + "id": "R7", + "prompt": "do {project}", + "tags": {"read"}, + "needs": {"items"}, + "verify": verify_ok, + } + + assert asyncio.run(run_live([task], model_alias="standard", reps=2, label="local", out_path=out)) == 0 + rows = _data_rows(out) + assert len({row["run_id"] for row in rows}) == 1 + assert [row["fixture_seed_id"] for row in rows] == [seed_id for seed_id, _ in seeded] + assert seeded[0][0] != seeded[1][0] + assert seeded[0][1] != seeded[1][1] def test_run_live_cleanup_failure_is_incomplete_without_changing_success(tmp_path, monkeypatch, capsys): diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py index 9d8de88f..0e81ae1b 100644 --- a/tests/evals/runner/test_resume.py +++ b/tests/evals/runner/test_resume.py @@ -47,8 +47,9 @@ def _should_skip_resume_row_non_null_error_retries(): def _should_skip_resume_row_only_plan_gated_skip_is_terminal(): - assert should_skip_resume_row({"skipped": "env:plan-gated:customers"}) is True - assert should_skip_resume_row({"skipped": "env:no-activity-worker"}) is True + assert should_skip_resume_row({"task_id": "L4", "skipped": "env:plan-gated:customers"}) is True + assert should_skip_resume_row({"task_id": "L2", "skipped": "env:no-activity-worker"}) is True + assert should_skip_resume_row({"task_id": "W1", "skipped": "env:plan-gated:customers"}) is False assert should_skip_resume_row({"skipped": "env:no-activity-worker (worker disabled)"}) is False assert should_skip_resume_row({"skipped": "env:plan-gated:customerz"}) is False assert should_skip_resume_row({"skipped": "env:fixture-collision:customers:Acme Corp"}) is False @@ -269,7 +270,7 @@ def test_run_live_resume_retries_infra_and_unexpected_skips_but_not_plan_gates(t "success": False, }, { - "task_id": "C1", + "task_id": "L4", "rep": 0, "label": "local", "driver": "claude-cli", @@ -286,6 +287,15 @@ def test_run_live_resume_retries_infra_and_unexpected_skips_but_not_plan_gates(t }, ] out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") + original_bytes = out.read_bytes() + original_write_text = Path.write_text + + def reject_results_rewrite(path, *args, **kwargs): + if path == out: + raise AssertionError("resume must not rewrite its append-only results file") + return original_write_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", reject_results_rewrite) fake_plane = MagicMock() monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) @@ -331,7 +341,7 @@ async def verify_ok(plane, ctx, run): "verify": verify_ok, }, { - "id": "C1", + "id": "L4", "prompt": "do {project}", "tags": set(), "needs": set(), @@ -358,10 +368,11 @@ async def verify_ok(plane, ctx, run): ) ) assert rc == 0 - # R1 completed and C1 was plan-gated. R2 infra and C2 collision are retried. + # R1 completed and L4 was plan-gated. R2 infra and C2 collision are retried. assert seed_calls == ["R2", "C2"] + assert out.read_bytes().startswith(original_bytes) data = _data_rows(out) - # Prior four + two retries (meta may also exist if file was empty — it wasn't). + # Resume is append-only: both retryable failures remain before their replacements. assert len(data) == 6 new_r2, new_c2 = data[-2:] assert new_r2["task_id"] == "R2" @@ -382,9 +393,13 @@ def test_make_run_meta_row_and_write_once(tmp_path: Path): model="sonnet", driver="claude-cli", git_sha="deadbeef", + expected_task_ids=["R1", "W1"], + expected_reps=3, ts="2026-01-01T00:00:00+00:00", ) assert meta["row_type"] == "meta" + assert meta["expected_task_ids"] == ["R1", "W1"] + assert meta["expected_reps"] == 3 assert is_meta_row(meta) assert is_meta_or_non_task_row(meta) assert maybe_write_run_meta(path, meta) is True @@ -396,3 +411,19 @@ def test_make_run_meta_row_and_write_once(tmp_path: Path): assert len(lines) == 2 assert json.loads(lines[0])["row_type"] == "meta" assert json.loads(lines[1])["task_id"] == "R1" + + +def test_make_run_meta_row_rejects_count_disagreement_with_exact_keys(): + with pytest.raises(ValueError, match="exact expectation=4"): + make_run_meta_row( + run_id="rid", + label="candidate", + server="local", + battery="abcd1234ef00", + model="sonnet", + driver="api", + git_sha="deadbeef", + expected_rows=3, + expected_task_ids=["R1", "W1"], + expected_reps=2, + ) diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py index b2c673d0..dd8c8828 100644 --- a/tests/evals/test_results.py +++ b/tests/evals/test_results.py @@ -169,6 +169,7 @@ def test_api_driver_maps_every_current_row_field(): assert row["model"] == "fake-actual" assert row["requested_model"] == "fake-requested" assert row["provider_stop_reason"] == "fake_done" + assert row["tool_manifest_fingerprint"] def _agent_run_dict_keeps_action_arg(): @@ -325,11 +326,16 @@ def test_agent_run_behaviours(case): def test_task_result_schema_round_trip_owns_usage_shape(): result = TaskResult( row_type="result", + run_id="run-1", + fixture_seed_id="fixture-seed-1", task_id="R1", + task_fingerprint="taskhash0001", label="local", server="local", expected_rows=35, cleanup_error="RuntimeError: teardown failed", + seeded_entity_kinds=["project", "work_item"], + randomized_seed_namespaces=["R2.urgent_open_count"], calls=[ CallRecord( tool="find_work_items", @@ -341,24 +347,40 @@ def test_task_result_schema_round_trip_owns_usage_shape(): ], num_calls=1, evidence_trace_available=True, + trace_integrity=False, + trace_integrity_reason="protocol_violation", + tool_manifest_fingerprint="manifest-sha256", usage_per_iteration=[Usage(10, 2, 3, 4)], ) row = result.to_row() assert row["schema_version"] == RESULT_SCHEMA_VERSION assert row["row_type"] == "result" + assert row["run_id"] == "run-1" + assert row["fixture_seed_id"] == "fixture-seed-1" + assert row["task_fingerprint"] == "taskhash0001" assert row["label"] == "local" assert row["server"] == "local" assert row["expected_rows"] == 35 assert row["cleanup_error"] == "RuntimeError: teardown failed" + assert row["seeded_entity_kinds"] == ["project", "work_item"] + assert row["randomized_seed_namespaces"] == ["R2.urgent_open_count"] assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] loaded = TaskResult.from_row(row) assert loaded.row_type == "result" + assert loaded.run_id == "run-1" + assert loaded.fixture_seed_id == "fixture-seed-1" + assert loaded.task_fingerprint == "taskhash0001" assert loaded.calls[0].tool == "find_work_items" assert loaded.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] assert loaded.evidence_trace_available is True + assert loaded.trace_integrity is False + assert loaded.trace_integrity_reason == "protocol_violation" + assert loaded.tool_manifest_fingerprint == "manifest-sha256" assert loaded.expected_rows == 35 assert loaded.cleanup_error == "RuntimeError: teardown failed" + assert loaded.seeded_entity_kinds == result.seeded_entity_kinds + assert loaded.randomized_seed_namespaces == result.randomized_seed_namespaces assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] diff --git a/tests/evals/test_skip_taxonomy.py b/tests/evals/test_skip_taxonomy.py index 42b7df42..474b7526 100644 --- a/tests/evals/test_skip_taxonomy.py +++ b/tests/evals/test_skip_taxonomy.py @@ -52,3 +52,13 @@ def test_plan_gated_capability_allowlist_matches_reviewed_seed_surfaces(): ) for capability in PLAN_GATED_CAPABILITIES: assert classify_skip_reason(f"env:plan-gated:{capability}") == "expected-capability", capability + + +def test_task_capability_pairs_are_derived_from_fixture_needs_and_fail_closed(): + assert classify_skip_reason("env:plan-gated:customers", task_id="L4") == "expected-capability" + assert classify_skip_reason("env:plan-gated:customers", task_id="W1") == "unexpected" + assert classify_skip_reason("env:plan-gated:releases", task_id="C2") == "expected-capability" + assert classify_skip_reason("env:plan-gated:releases", task_id="L3") == "unexpected" + assert classify_skip_reason("env:plan-gated:work-item-types", task_id="S1") == "expected-capability" + assert classify_skip_reason("env:no-activity-worker", task_id="L2") == "expected-capability" + assert classify_skip_reason("env:no-activity-worker", task_id="R1") == "unexpected" From b9eb079c0ccc8cbb8b29830de6a6bab71b534e23 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sat, 15 Aug 2026 22:23:59 +0530 Subject: [PATCH 42/93] Record a trace the CLI can actually produce The recording proxy had never finalized. write_meta ran only from a finally block and no signal handler was installed, so the SIGTERM the CLI sends its MCP child skipped it: measured, SIGINT wrote proxy_meta and SIGTERM wrote nothing. Every trace this harness ever recorded was missing its footer, which is why pending_left, last_seq, unparsed_lines, unmatched_responses and the tool manifest were never observable, and why no_meta appeared on 100% of rows in the last two batteries. The handler requests a controlled drain and lets the existing finally write; calling write_meta from the handler itself would deadlock against the recorder's non-reentrant lock. The driver also harvested before the proxy finalized. wait_for_proxy_meta existed and was wired only into the CLI-timeout path, so every normal completion read the sidecar early. The wait is now centralized, polls, and costs 0.04ms when the trace is already final. With finalization working, the CLI turned out to start the MCP server twice per task - an enumeration probe, then the session - so one sidecar holds several proxy sessions. Sessions are now parsed as segments terminated by their own proxy_meta and validated independently: seq continuity is per session, and several metas is normal where none is still loss. Validating sequences across the whole file would have reported a phantom duplicate the first time a probe made a call. The evidence configuration was one-shot, so the probe session consumed and unlinked it and the real session ran with matching disabled. Read provenance was silently dead on every CLI run, failing correct agents. The file is now readable for the run's lifetime, which exposes nothing new: it carries target ids and one-way fingerprints, never sentinel values, and the driver's temporary directory still removes it. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 5 +- evals/README.md | 7 +- evals/drivers/__init__.py | 2 + evals/drivers/cli/sidecar.py | 381 +++++++++++++++++-------- evals/evidence.py | 14 +- evals/proxy.py | 99 ++++++- tests/evals/drivers/test_cli_driver.py | 146 +++++++++- tests/evals/test_proxy.py | 328 +++++++++++++++++++-- 8 files changed, 823 insertions(+), 159 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 661c9c2e..7717b2aa 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -128,8 +128,9 @@ hidden, per-run sentinel and its seeded target entity ID. At the API loop or CLI proxy, the harness requires the request arguments to target that ID and matches the sentinel while the successful response is in memory, persisting only a non-sensitive `observed_sentinels` label. CLI proxies receive only target IDs plus sentinel lengths and -SHA-256 fingerprints through a mode-0600 one-shot file outside the agent cwd; the raw value -is absent even from that file. They consume and unlink it before starting Plane MCP. A successful response from +SHA-256 fingerprints through a mode-0600 run-scoped file outside the agent cwd; the raw value +is absent even from that file. Every MCP proxy session reads the same file, and the driver +removes it with its temporary directory after the run. A successful response from an unrelated entity therefore does not count, and neither the response body nor sentinel enters the payload-free result row. Unavailable or incomplete matching is diagnosed and cannot pass a read verifier. diff --git a/evals/README.md b/evals/README.md index 1fa92200..302c10ab 100644 --- a/evals/README.md +++ b/evals/README.md @@ -115,9 +115,10 @@ habit; use it only when the more sensitive, larger sidecar is justified. Read-task provenance is stricter than “a call happened.” Seeders place a hidden per-run sentinel on the target entity, and the API driver or CLI proxy records only whether a successful response exposed it **and its request targeted that seeded entity ID**. CLI -proxies receive only target IDs and one-way value fingerprints through a private one-shot -file; the raw sentinel is absent even if that file is inspected. The proxy unlinks it before -Plane MCP starts. Result rows contain the matched label, never the sentinel value or response body. Thus an unrelated +proxies receive only target IDs and one-way value fingerprints through a private run-scoped +file; the raw sentinel is absent even if that file is inspected. Every proxy session can read it, and the driver +removes it with its temporary directory after the run. Result rows contain the matched label, never the sentinel +value or response body. Thus an unrelated successful response cannot satisfy provenance; unavailable or incomplete matching is diagnosed and fails closed. diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index d263ba32..686a8d67 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -39,6 +39,7 @@ harvest_proxy_after_cli_timeout, load_proxy_sidecar, load_proxy_sidecar_calls, + proxy_pid_path, proxy_wrap_server_command, wait_for_proxy_meta, ) @@ -91,6 +92,7 @@ def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: "parse_codex_jsonl_events", "parse_codex_rollout_calls", "prepare_antigravity_fake_home", + "proxy_pid_path", "proxy_wrap_server_command", "run_cli_subprocess", "wait_for_proxy_meta", diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 56439a53..4dc3e2e1 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -9,11 +9,13 @@ from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Literal from evals import REPO_ROOT from evals.results import TraceIntegrityReason +ProxyMetaWaitOutcome = Literal["meta_present", "proxy_exited", "proxy_not_observed", "timeout"] + @dataclass(slots=True) class ProxySidecarResult: @@ -76,22 +78,27 @@ def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: def load_proxy_sidecar( path: Path, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Load sidecar call rows (sorted by seq) plus a status dict. + """Load and validate the sidecar as independently finalized proxy sessions. Status keys: - missing / empty / complete / incomplete - torn_line: final line failed to parse - skipped_rows: non-final rows that could not produce a call or metadata row - - meta: proxy_meta row if present - - pending_left / non_tool_pending_left: unmatched requests from meta - - sequence_errors: invalid, duplicate, missing, or unexpected call sequence values - - proxy_meta_count / proxy_meta_not_final: metadata framing integrity + - meta / metas / sessions: final metadata plus per-session validation + - pending_left / non_tool_pending_left: sums across sessions + - sequence errors: sums of per-session invalid/duplicate/missing/unexpected values + - proxy_meta_not_final: a trailing session lacks its terminating metadata + - tool_manifest_disagreement: finalized sessions advertised different surfaces """ status: dict[str, Any] = { "state": "missing", "torn_line": False, "skipped_rows": 0, "meta": None, + "metas": [], + "sessions": [], + "session_count": 0, + "unfinalized_sessions": 0, "pending_left": None, "non_tool_pending_left": None, "proxy_meta_count": 0, @@ -101,6 +108,9 @@ def load_proxy_sidecar( "missing_seq": 0, "unexpected_seq": 0, "invalid_meta_fields": 0, + "tool_manifest_disagreement": False, + "tool_manifest_fingerprints": [], + "tool_manifest_missing_sessions": 0, } if not path.is_file(): return [], status @@ -112,39 +122,39 @@ def load_proxy_sidecar( status["state"] = "empty" return [], status - # Decode with replacement so invalid UTF-8 does not crash the loader. - text = raw.decode("utf-8", errors="replace") - lines = text.splitlines() - calls: list[dict[str, Any]] = [] - meta: dict[str, Any] | None = None - meta_positions: list[int] = [] - nonblank_position = 0 - torn = False - skipped_rows = 0 + # Each proxy process restarts seq at 1 and terminates its segment with + # proxy_meta. Validate and order calls within those boundaries, never over + # the whole file where healthy sessions can have colliding seq values. + lines = raw.decode("utf-8", errors="replace").splitlines() + raw_segments: list[dict[str, Any]] = [] + current: dict[str, Any] = {"calls": [], "meta": None, "torn_line": False, "skipped_rows": 0} + current_has_rows = False for i, line in enumerate(lines): s = line.strip() if not s: continue - nonblank_position += 1 + current_has_rows = True try: row = json.loads(s) except json.JSONDecodeError: # Tolerate a torn final line (crash mid-write); stop there. if i == len(lines) - 1: - torn = True + current["torn_line"] = True break - skipped_rows += 1 + current["skipped_rows"] += 1 continue if not isinstance(row, dict): - skipped_rows += 1 + current["skipped_rows"] += 1 continue if row.get("row_type") == "proxy_meta": - meta = row - meta_positions.append(nonblank_position) + current["meta"] = row + raw_segments.append(current) + current = {"calls": [], "meta": None, "torn_line": False, "skipped_rows": 0} + current_has_rows = False continue tool = row.get("tool") if not tool: - skipped_rows += 1 + current["skipped_rows"] += 1 continue call = { "tool": str(tool), @@ -160,60 +170,23 @@ def load_proxy_sidecar( call["result_text"] = row["result_text"] if isinstance(row.get("observed_sentinels"), list): call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] - calls.append(call) - - valid_sequences = [call["seq"] for call in calls if _nonnegative_int(call.get("seq")) not in (None, 0)] - invalid_seq = len(calls) - len(valid_sequences) - duplicate_seq = len(valid_sequences) - len(set(valid_sequences)) - last_seq = _nonnegative_int(meta.get("last_seq")) if meta is not None else None - tool_request_count = _nonnegative_int(meta.get("tool_request_count")) if meta is not None else None - invalid_meta_fields = int(meta is not None and last_seq is None) + int( - meta is not None and tool_request_count is None - ) - if last_seq is not None and tool_request_count is not None and tool_request_count != last_seq: - invalid_meta_fields += 1 - expected_sequences = set(range(1, last_seq + 1)) if last_seq is not None else set() - observed_sequences = set(valid_sequences) - missing_seq = len(expected_sequences - observed_sequences) - unexpected_seq = len(observed_sequences - expected_sequences) if last_seq is not None else 0 - - # Score order must match request seq, not response-append order. Invalid seq - # rows remain available for diagnostics but can never make the trace valid. - calls.sort( - key=lambda call: ( - _nonnegative_int(call.get("seq")) in (None, 0), - _nonnegative_int(call.get("seq")) or 0, - ) - ) + current["calls"].append(call) - status["torn_line"] = torn - status["skipped_rows"] = skipped_rows - status["meta"] = meta - status["proxy_meta_count"] = len(meta_positions) - status["proxy_meta_not_final"] = bool(meta_positions and meta_positions[-1] != nonblank_position) - status["invalid_seq"] = invalid_seq - status["duplicate_seq"] = duplicate_seq - status["missing_seq"] = missing_seq - status["unexpected_seq"] = unexpected_seq - status["invalid_meta_fields"] = invalid_meta_fields - if meta is not None: - counter_keys = ( - "pending_left", - "non_tool_pending_left", - "unmatched_responses", - "unparsed_lines", - "non_json_lines", - "malformed_jsonrpc", - "recorder_errors", - ) - for key in counter_keys: - value = _nonnegative_int(meta.get(key)) - if meta.get(key) is not None and value is None: - status["invalid_meta_fields"] += 1 - status[key] = value - status["pumps_alive"] = bool(meta.get("pumps_alive")) - fingerprint = meta.get("tool_manifest_fingerprint") - status["tool_manifest_fingerprint"] = str(fingerprint) if isinstance(fingerprint, str) else None + if current_has_rows: + raw_segments.append(current) + if not raw_segments: + status["state"] = "empty" + return [], status + + counter_keys = ( + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "non_json_lines", + "malformed_jsonrpc", + "recorder_errors", + ) fatal_counts = ( "pending_left", "non_tool_pending_left", @@ -226,20 +199,107 @@ def load_proxy_sidecar( "unexpected_seq", "invalid_meta_fields", ) - incomplete = bool( - torn - or skipped_rows > 0 - or len(meta_positions) != 1 - or status["proxy_meta_not_final"] - or any((status.get(key) or 0) > 0 for key in fatal_counts) - or (meta is not None and bool(meta.get("pumps_alive"))) + calls: list[dict[str, Any]] = [] + session_statuses: list[dict[str, Any]] = [] + manifests: list[str] = [] + for index, raw_segment in enumerate(raw_segments): + segment_calls = raw_segment["calls"] + meta = raw_segment["meta"] + valid_sequences = [call["seq"] for call in segment_calls if _nonnegative_int(call.get("seq")) not in (None, 0)] + last_seq = _nonnegative_int(meta.get("last_seq")) if meta is not None else None + tool_request_count = _nonnegative_int(meta.get("tool_request_count")) if meta is not None else None + segment: dict[str, Any] = { + "index": index, + "meta": meta, + "call_count": len(segment_calls), + "torn_line": bool(raw_segment["torn_line"]), + "skipped_rows": int(raw_segment["skipped_rows"]), + "invalid_seq": len(segment_calls) - len(valid_sequences), + "duplicate_seq": len(valid_sequences) - len(set(valid_sequences)), + "missing_seq": 0, + "unexpected_seq": 0, + "invalid_meta_fields": 0, + "pumps_alive": bool(meta.get("pumps_alive")) if meta is not None else False, + "last_seq": last_seq, + "tool_request_count": tool_request_count, + } + if meta is not None: + segment["invalid_meta_fields"] = int(last_seq is None) + int(tool_request_count is None) + if last_seq is not None and tool_request_count is not None and tool_request_count != last_seq: + segment["invalid_meta_fields"] += 1 + expected_sequences = set(range(1, last_seq + 1)) if last_seq is not None else set() + observed_sequences = set(valid_sequences) + segment["missing_seq"] = len(expected_sequences - observed_sequences) + segment["unexpected_seq"] = len(observed_sequences - expected_sequences) if last_seq is not None else 0 + for key in counter_keys: + value = _nonnegative_int(meta.get(key)) + if meta.get(key) is not None and value is None: + segment["invalid_meta_fields"] += 1 + segment[key] = value + fingerprint = meta.get("tool_manifest_fingerprint") + if isinstance(fingerprint, str): + manifests.append(fingerprint) + else: + for key in counter_keys: + segment[key] = None + + segment["state"] = ( + "incomplete" + if meta is None + or segment["torn_line"] + or segment["skipped_rows"] > 0 + or any((segment.get(key) or 0) > 0 for key in fatal_counts) + or segment["pumps_alive"] + else "complete" + ) + # Preserve request order inside a session, then concatenate sessions in + # file order. Never globally sort colliding per-process seq values. + segment_calls.sort( + key=lambda call: ( + _nonnegative_int(call.get("seq")) in (None, 0), + _nonnegative_int(call.get("seq")) or 0, + ) + ) + calls.extend(segment_calls) + session_statuses.append(segment) + + metas = [segment["meta"] for segment in session_statuses if segment["meta"] is not None] + unfinalized_sessions = sum(segment["meta"] is None for segment in session_statuses) + status["sessions"] = session_statuses + status["session_count"] = len(session_statuses) + status["metas"] = metas + status["meta"] = metas[-1] if metas else None + status["proxy_meta_count"] = len(metas) + status["unfinalized_sessions"] = unfinalized_sessions + status["proxy_meta_not_final"] = unfinalized_sessions > 0 + status["torn_line"] = any(segment["torn_line"] for segment in session_statuses) + status["pumps_alive"] = any(segment["pumps_alive"] for segment in session_statuses) + aggregate_keys = ( + "skipped_rows", + *counter_keys, + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ) + for key in aggregate_keys: + status[key] = sum((segment.get(key) or 0) for segment in session_statuses) + + unique_manifests = sorted(set(manifests)) + missing_manifests = len(metas) - len(manifests) + status["tool_manifest_fingerprints"] = unique_manifests + status["tool_manifest_missing_sessions"] = missing_manifests + status["tool_manifest_disagreement"] = len(unique_manifests) > 1 or bool(unique_manifests and missing_manifests) + status["tool_manifest_fingerprint"] = ( + unique_manifests[0] if len(unique_manifests) == 1 and missing_manifests == 0 else None + ) + if status["meta"] is not None: + status["finalization_reason"] = status["meta"].get("finalization_reason") + status["finalization_signal"] = status["meta"].get("finalization_signal") + status["state"] = ( + "incomplete" if any(segment["state"] == "incomplete" for segment in session_statuses) else "complete" ) - if not calls and not meta and not torn and skipped_rows == 0: - status["state"] = "empty" - elif incomplete: - status["state"] = "incomplete" - else: - status["state"] = "complete" return calls, status @@ -258,10 +318,15 @@ def _incompleteness_note(status: dict[str, Any]) -> str: parts = ["proxy_sidecar_incomplete"] if status.get("torn_line"): parts.append("torn_line=1") + if status.get("meta") is None: + parts.append("no_meta=1") + else: + if status.get("proxy_meta_not_final"): + parts.append("proxy_meta_not_final=1") + if status.get("unfinalized_sessions"): + parts.append(f"unfinalized_sessions={int(status['unfinalized_sessions'])}") for key in ( "skipped_rows", - "proxy_meta_count", - "proxy_meta_not_final", "pending_left", "non_tool_pending_left", "unmatched_responses", @@ -278,8 +343,6 @@ def _incompleteness_note(status: dict[str, Any]) -> str: value = status.get(key) if value and not (key == "proxy_meta_count" and value == 1): parts.append(f"{key}={int(value)}") - if status.get("meta") is None: - parts.append("no_meta=1") if status.get("pumps_alive"): parts.append("pumps_alive=1") return ":".join(parts) @@ -291,22 +354,118 @@ def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: return calls +def proxy_pid_path(sidecar_path: Path) -> Path: + """Return the companion lifecycle file written by the recording proxy.""" + return sidecar_path.with_name(f"{sidecar_path.name}.pid") + + +def _read_proxy_pid(sidecar_path: Path) -> int | None: + try: + raw = proxy_pid_path(sidecar_path).read_text(encoding="ascii").strip() + pid = int(raw) + except (OSError, UnicodeError, ValueError): + return None + return pid if pid > 0 else None + + +def _process_is_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _wait_for_proxy_meta_outcome( + sidecar_path: Path, + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> ProxyMetaWaitOutcome: + """Wait for final metadata and retain why an absent row cannot still arrive.""" + # Local import keeps drivers import-light for non-proxy unit tests. + from evals.proxy import SHUTDOWN_DEADLINE_S + + if max_wait_s is None: + max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 + + # A completed CLI cannot launch a proxy after the fact. No trace and no + # lifecycle file therefore means the proxy was never observed, rather than + # a finalizer that could benefit from waiting for the whole deadline. + pid = _read_proxy_pid(sidecar_path) + if not sidecar_path.exists() and pid is None: + return "proxy_not_observed" + + deadline = time.monotonic() + max(0.0, max_wait_s) + while True: + _, status = load_proxy_sidecar(sidecar_path) + if status.get("meta") is not None: + return "meta_present" + pid = _read_proxy_pid(sidecar_path) + if pid is not None and not _process_is_alive(pid): + return "proxy_exited" + rem = deadline - time.monotonic() + if rem <= 0: + break + time.sleep(min(max(0.001, poll_s), rem)) + + # Close the boundary races in both directions: metadata may have landed on + # the final sleep, or the proxy may have exited without writing it. + _, status = load_proxy_sidecar(sidecar_path) + if status.get("meta") is not None: + return "meta_present" + pid = _read_proxy_pid(sidecar_path) + if pid is not None and not _process_is_alive(pid): + return "proxy_exited" + return "timeout" + + +def _note_proxy_meta_wait(outcome: ProxyMetaWaitOutcome, sidecar_path: Path, notes: list[str]) -> None: + if outcome == "proxy_exited": + notes.append("proxy_meta_missing_after_proxy_exit") + elif outcome == "proxy_not_observed": + notes.append("proxy_meta_missing:proxy_not_observed") + elif outcome == "timeout": + pid = _read_proxy_pid(sidecar_path) + state = "proxy_alive=1" if pid is not None and _process_is_alive(pid) else "proxy_state=unknown" + notes.append(f"proxy_meta_wait_timeout:{state}") + + def apply_proxy_sidecar( calls: list[dict[str, Any]], client_calls: list[dict[str, Any]], sidecar_path: Path, notes: list[str], + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, ) -> ProxySidecarResult: - """Prefer a complete proxy sidecar; fall back to CLI-parsed when incomplete/empty. + """Wait for and prefer a complete sidecar; fall back when incomplete/empty. Incomplete sidecar (torn/skipped row, missing meta, pending_left>0) yields to the CLI trace when the CLI has *more* plane calls. Returns ``(plane_calls, client_calls, call_source)``. """ + wait_outcome = _wait_for_proxy_meta_outcome( + sidecar_path, + poll_s=poll_s, + max_wait_s=max_wait_s, + ) + _note_proxy_meta_wait(wait_outcome, sidecar_path, notes) proxy_calls, status = load_proxy_sidecar(sidecar_path) state = status.get("state") trace_integrity, trace_integrity_reason = trace_integrity_from_status(status) fingerprint = status.get("tool_manifest_fingerprint") if trace_integrity else None + if status.get("tool_manifest_disagreement"): + manifest_values = list(status.get("tool_manifest_fingerprints") or []) + if status.get("tool_manifest_missing_sessions"): + manifest_values.append("") + manifests = ",".join(manifest_values) + notes.append(f"proxy_tool_manifest_disagreement:{manifests}") def result( selected_calls: list[dict[str, Any]], @@ -352,22 +511,14 @@ def wait_for_proxy_meta( EOF and needs up to SHUTDOWN_DEADLINE_S to flush. Call before harvesting so the temp directory is not deleted mid-finalization. """ - # Local import keeps drivers import-light for non-proxy unit tests. - from evals.proxy import SHUTDOWN_DEADLINE_S - - if max_wait_s is None: - max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 - deadline = time.monotonic() + max_wait_s - while True: - _, status = load_proxy_sidecar(sidecar_path) - if status.get("meta") is not None: - return True - rem = deadline - time.monotonic() - if rem <= 0: - break - time.sleep(min(poll_s, rem)) - _, status = load_proxy_sidecar(sidecar_path) - return status.get("meta") is not None + return ( + _wait_for_proxy_meta_outcome( + sidecar_path, + poll_s=poll_s, + max_wait_s=max_wait_s, + ) + == "meta_present" + ) def harvest_proxy_after_cli_timeout( @@ -384,10 +535,13 @@ def harvest_proxy_after_cli_timeout( note from ``apply_proxy_sidecar``). ``max_wait_s`` defaults to ``SHUTDOWN_DEADLINE_S + 2`` (see ``wait_for_proxy_meta``). """ - found = wait_for_proxy_meta(sidecar_path, max_wait_s=max_wait_s) - if not found: - notes.append("proxy_meta_wait_timeout") - return apply_proxy_sidecar(calls, client_calls, sidecar_path, notes) + return apply_proxy_sidecar( + calls, + client_calls, + sidecar_path, + notes, + max_wait_s=max_wait_s, + ) __all__ = [ @@ -396,6 +550,7 @@ def harvest_proxy_after_cli_timeout( "harvest_proxy_after_cli_timeout", "load_proxy_sidecar", "load_proxy_sidecar_calls", + "proxy_pid_path", "proxy_wrap_server_command", "ProxySidecarResult", "trace_integrity_from_status", diff --git a/evals/evidence.py b/evals/evidence.py index 851c5b42..c2b56df4 100644 --- a/evals/evidence.py +++ b/evals/evidence.py @@ -194,7 +194,7 @@ def decode_evidence_config( def write_evidence_config(path: Path, sentinels: Any, targets: Any, aggregates: Any = None) -> None: - """Create a private, one-shot proxy configuration outside the agent cwd.""" + """Create a private, run-scoped proxy configuration outside the agent cwd.""" payload = encode_evidence_config(sentinels, targets, aggregates) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as stream: @@ -208,18 +208,18 @@ def consume_evidence_config( dict[str, tuple[str, ...]], dict[str, tuple[dict[str, Any], ...]], ]: - """Read and unlink a one-shot proxy configuration, failing closed.""" + """Read a reusable run-scoped proxy configuration, failing closed. + + The historical name is retained for callers. A CLI may start multiple MCP + proxy sessions during one task, so the driver's TemporaryDirectory owns + deletion after every session has exited. + """ if path is None: return {}, {}, {} try: raw = path.read_text(encoding="utf-8") except OSError: return {}, {}, {} - finally: - try: - path.unlink() - except OSError: - pass return decode_evidence_config(raw) diff --git a/evals/proxy.py b/evals/proxy.py index 5e025760..3a4053c3 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -12,10 +12,12 @@ import json import os import select +import signal import subprocess import sys import threading import time +from collections.abc import Callable from pathlib import Path from typing import Any @@ -61,7 +63,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p.add_argument( "--evidence-file", type=Path, - help="One-shot target-evidence configuration consumed before the MCP child starts", + help="Run-scoped target-evidence configuration loaded before the MCP child starts", ) p.add_argument( "command", @@ -173,6 +175,8 @@ def __init__( self.server_requests = 0 self.child_killed = False self.pumps_alive = False + self.finalization_reason = "direct" + self.finalization_signal: str | None = None self.finalized = False # Post-finalize append attempts (not written; for tests / diagnostics). self.post_finalize_appends = 0 @@ -349,6 +353,8 @@ def write_meta(self) -> None: "tool_request_count": self._seq, "child_killed": self.child_killed, "pumps_alive": self.pumps_alive, + "finalization_reason": self.finalization_reason, + "finalization_signal": self.finalization_signal, "evidence_trace_available": self.evidence_active, "tool_manifest_fingerprint": self._tool_manifest.fingerprint, } @@ -540,6 +546,18 @@ def reap_timeout(deadline_at: float | None, floor: float = 0.1) -> float: return max(floor, _remaining(deadline_at)) +def _signal_name(signum: int) -> str: + try: + return signal.Signals(signum).name + except ValueError: + return str(signum) + + +def _record_signal_finalization(recorder: SidecarRecorder, signum: int) -> None: + recorder.finalization_reason = "signal" + recorder.finalization_signal = _signal_name(signum) + + def run_proxy( command: list[str], log_path: Path, @@ -549,6 +567,7 @@ def run_proxy( evidence_fingerprints: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, evidence_aggregates: dict[str, Any] | None = None, + termination_signal: Callable[[], int | None] | None = None, ) -> int: """Spawn ``command`` as the real MCP server and relay with recording. @@ -565,6 +584,16 @@ def run_proxy( evidence_targets=evidence_targets, evidence_aggregates=evidence_aggregates, ) + recorder.finalization_reason = "running" + # The CLI driver does not own this detached process, so leave a companion + # lifecycle file that lets it distinguish a slow finalizer from a proxy + # that exited before writing proxy_meta. The temp directory owns cleanup. + try: + log_path.with_name(f"{log_path.name}.pid").write_text(str(os.getpid()), encoding="ascii") + except OSError: + # Metadata remains authoritative. A missing lifecycle file merely + # leaves timeout diagnostics with an unknown process state. + pass child: subprocess.Popen[bytes] | None = None # Scrub repo PYTHONPATH so the real server does not import from this tree. child_env = scrub_child_pythonpath() @@ -582,6 +611,7 @@ def run_proxy( env=child_env, ) except OSError as exc: + recorder.finalization_reason = "spawn_failure" print(f"evals.proxy: failed to spawn {command!r}: {exc}", file=sys.stderr) return 1 @@ -649,9 +679,21 @@ def run_proxy( # Phase 1: run until client stdin EOF or child exits. # Child exit cancels the *stdin* pump only — stdout must drain to pipe EOF. - while child.poll() is None and not stdin_done.is_set(): + while ( + child.poll() is None + and not stdin_done.is_set() + and (termination_signal is None or termination_signal() is None) + ): time.sleep(0.05) + requested_signal = termination_signal() if termination_signal is not None else None + if requested_signal is not None: + _record_signal_finalization(recorder, requested_signal) + elif child.poll() is not None: + recorder.finalization_reason = "child_exit" + else: + recorder.finalization_reason = "normal_eof" + # One deadline for the entire post-EOF / post-child-exit shutdown. deadline_at = time.monotonic() + SHUTDOWN_DEADLINE_S cancel_stdin.set() @@ -712,6 +754,14 @@ def run_proxy( ) recorder.pumps_alive = pumps_still return map_child_returncode(child.returncode) + except KeyboardInterrupt: + # Preserve Python's existing SIGINT behaviour: unwind through the + # finalizer, then let KeyboardInterrupt retain the signal exit status. + _record_signal_finalization(recorder, signal.SIGINT) + raise + except BaseException: + recorder.finalization_reason = "exception" + raise finally: if child is not None and child.poll() is None: try: @@ -729,6 +779,9 @@ def run_proxy( still = any(t is not None and t.is_alive() for t in (t_in, t_out, t_err)) if still: recorder.pumps_alive = True + requested_signal = termination_signal() if termination_signal is not None else None + if requested_signal is not None: + _record_signal_finalization(recorder, requested_signal) try: recorder.write_meta() except Exception as exc: @@ -749,14 +802,40 @@ def main(argv: list[str] | None = None) -> int: pass args = parse_args(argv) evidence_fingerprints, evidence_targets, evidence_aggregates = consume_evidence_config(args.evidence_file) - return run_proxy( - list(args.command), - Path(args.log), - record_result_payloads=bool(args.record_result_payloads), - evidence_fingerprints=evidence_fingerprints, - evidence_targets=evidence_targets, - evidence_aggregates=evidence_aggregates, - ) + received_signal: list[int | None] = [None] + + def request_termination(signum: int, _frame: Any) -> None: + # Do not finalize in the handler: it can interrupt code holding the + # recorder lock. The relay loop observes this state and drains first. + if received_signal[0] is None: + received_signal[0] = signum + + previous_handlers: dict[int, Any] = {} + for signum in (signal.SIGTERM, signal.SIGHUP): + previous_handlers[signum] = signal.getsignal(signum) + signal.signal(signum, request_termination) + try: + returncode = run_proxy( + list(args.command), + Path(args.log), + record_result_payloads=bool(args.record_result_payloads), + evidence_fingerprints=evidence_fingerprints, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, + termination_signal=lambda: received_signal[0], + ) + finally: + for signum, previous_handler in previous_handlers.items(): + signal.signal(signum, previous_handler) + + signum = received_signal[0] + if signum is not None: + # Metadata and child cleanup are complete. Re-deliver with the + # default disposition so subprocess/shell status encodes the signal. + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + + return returncode if __name__ == "__main__": diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 1077793c..1a856d69 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -22,6 +22,7 @@ harvest_proxy_after_cli_timeout, load_proxy_sidecar, load_proxy_sidecar_calls, + proxy_pid_path, proxy_wrap_server_command, run_cli_subprocess, wait_for_proxy_meta, @@ -483,6 +484,7 @@ def _apply_proxy_sidecar_replaces_when_nonempty(tmp_path): [], side, notes, + max_wait_s=0, ) assert src == "proxy" assert calls[0]["tool"] == "find_work_items" @@ -495,7 +497,7 @@ def _apply_proxy_sidecar_empty_fallback(tmp_path): side.write_text("", encoding="utf-8") notes: list[str] = [] original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] - calls, _client, src = apply_proxy_sidecar(original, [], side, notes) + calls, _client, src = apply_proxy_sidecar(original, [], side, notes, max_wait_s=0) assert calls is original or calls == original assert "proxy_sidecar_empty" in notes assert src != "proxy" or calls == original @@ -523,7 +525,7 @@ def _apply_proxy_incomplete_defers_to_richer_cli(tmp_path): {"tool": "c2", "args": {}, "origin": "plane"}, ] notes: list[str] = [] - calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes, max_wait_s=0) assert src != "proxy" assert [c["tool"] for c in calls] == ["c1", "c2"] assert any("proxy_sidecar_incomplete" in n for n in notes) @@ -1056,6 +1058,144 @@ def test_wait_for_proxy_meta_unit(tmp_path: Path): assert wait_for_proxy_meta(side, max_wait_s=1.0, poll_s=0.05) is True +def test_normal_completion_waits_for_delayed_proxy_meta(tmp_path: Path): + """The shared normal path must not harvest the call row before final metadata.""" + import threading + + early_notes: list[str] = [] + meta_written = threading.Event() + writer_threads: list[threading.Thread] = [] + + def fake_run(cmd, **kwargs): + del kwargs + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + config = json.loads(config_path.read_text(encoding="utf-8")) + proxy_args = config["mcpServers"]["plane"]["args"] + sidecar = Path(proxy_args[proxy_args.index("--log") + 1]) + call = { + "tool": "delayed_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + sidecar.write_text(json.dumps(call) + "\n", encoding="utf-8") + proxy_pid_path(sidecar).write_text(str(os.getpid()), encoding="ascii") + + # This is the historical harvest: the call exists, but finalization has + # not happened, so accepting it now manufactures recorder loss. + early = apply_proxy_sidecar([], [], sidecar, early_notes, max_wait_s=0) + assert early.trace_integrity is False + assert "proxy_sidecar_incomplete:no_meta=1" in early_notes + + def write_meta_later() -> None: + time.sleep(0.08) + meta = { + "row_type": "proxy_meta", + "pending_left": 0, + "non_tool_pending_left": 0, + "unmatched_responses": 0, + "unparsed_lines": 0, + "recorder_errors": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + with sidecar.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta) + "\n") + meta_written.set() + + writer = threading.Thread(target=write_meta_later, daemon=True) + writer_threads.append(writer) + writer.start() + output = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "delayed-meta-session", + "num_turns": 1, + } + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(output), stderr="") + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + for writer in writer_threads: + writer.join(timeout=1.0) + + assert meta_written.is_set() + assert run.trace_integrity is True + assert run.trace_integrity_reason is None + assert run.call_source == "proxy" + assert [call["tool"] for call in run.calls] == ["delayed_meta_tool"] + assert not any("no_meta" in note for note in run.notes) + + +def test_proxy_meta_wait_timeout_is_fatal_and_diagnosable(tmp_path: Path): + sidecar = tmp_path / "never-finalized.jsonl" + sidecar.write_text( + json.dumps({"tool": "unfinished", "args": {}, "seq": 1}) + "\n", + encoding="utf-8", + ) + proxy_pid_path(sidecar).write_text(str(os.getpid()), encoding="ascii") + notes: list[str] = [] + + result = apply_proxy_sidecar([], [], sidecar, notes, poll_s=0.005, max_wait_s=0.03) + + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert "proxy_meta_wait_timeout:proxy_alive=1" in notes + assert "proxy_sidecar_incomplete:no_meta=1" in notes + + +def test_proxy_exit_before_meta_is_fatal_and_diagnosable(tmp_path: Path): + sidecar = tmp_path / "exited-before-meta.jsonl" + sidecar.write_text( + json.dumps({"tool": "unfinished", "args": {}, "seq": 1}) + "\n", + encoding="utf-8", + ) + exited = subprocess.Popen([sys.executable, "-c", "pass"]) + exited.wait(timeout=2.0) + proxy_pid_path(sidecar).write_text(str(exited.pid), encoding="ascii") + notes: list[str] = [] + started = time.monotonic() + + result = apply_proxy_sidecar([], [], sidecar, notes, poll_s=0.01, max_wait_s=1.0) + + assert time.monotonic() - started < 0.2 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert "proxy_meta_missing_after_proxy_exit" in notes + assert "proxy_sidecar_incomplete:no_meta=1" in notes + + +def test_proxy_meta_fast_path_does_not_sleep(tmp_path: Path, monkeypatch): + sidecar = tmp_path / "already-finalized.jsonl" + meta = { + "row_type": "proxy_meta", + "pending_left": 0, + "last_seq": 0, + "tool_request_count": 0, + } + sidecar.write_text(json.dumps(meta) + "\n", encoding="utf-8") + monkeypatch.setattr("evals.drivers.cli.sidecar.time.sleep", lambda _seconds: pytest.fail("fast path slept")) + notes: list[str] = [] + started = time.monotonic() + + result = apply_proxy_sidecar([], [], sidecar, notes, max_wait_s=1.0) + + assert time.monotonic() - started < 0.1 + assert result.trace_integrity is True + assert not any("proxy_meta_wait" in note for note in notes) + + def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): """If meta never arrives, harvest still returns with incomplete note.""" side = tmp_path / "s.jsonl" @@ -1065,7 +1205,7 @@ def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): ) notes: list[str] = [] calls, _client, src = harvest_proxy_after_cli_timeout([], [], side, notes, max_wait_s=0.25) - assert "proxy_meta_wait_timeout" in notes + assert "proxy_meta_wait_timeout:proxy_state=unknown" in notes assert len(calls) == 1 assert src == "proxy" assert any("incomplete" in n for n in notes) diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 87a936ce..c67acbe2 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -4,10 +4,12 @@ import json import os +import signal import stat import subprocess import sys import textwrap +import time from pathlib import Path import pytest @@ -18,7 +20,12 @@ load_proxy_sidecar, load_proxy_sidecar_calls, ) -from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE, write_evidence_config +from evals.evidence import ( + EVIDENCE_SENTINELS_ENV, + TARGET_ENTITY_EVIDENCE, + consume_evidence_config, + write_evidence_config, +) from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -48,6 +55,32 @@ def _response(request_id: int, result: object) -> dict: return {"jsonrpc": "2.0", "id": request_id, "result": result} +def _complete_proxy_meta(last_seq: int, manifest: str = "manifest-a") -> dict: + return { + "row_type": "proxy_meta", + "pending_left": 0, + "non_tool_pending_left": 0, + "unmatched_responses": 0, + "unparsed_lines": 0, + "recorder_errors": 0, + "pumps_alive": False, + "last_seq": last_seq, + "tool_request_count": last_seq, + "tool_manifest_fingerprint": manifest, + } + + +def _proxy_call(tool: str, seq: int) -> dict: + return { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": seq, + } + + FAKE_SERVER = textwrap.dedent( r""" import json, sys @@ -164,6 +197,7 @@ def _proxy_records_tools_call_and_exit_code(tmp_path): timeout=15, ) assert proc.returncode == 7 # child exit propagated + assert int(sidecar.with_name(f"{sidecar.name}.pid").read_text(encoding="ascii")) > 0 # Byte-faithful: unparsed line and JSON responses appear on stdout. out = proc.stdout.decode("utf-8", errors="replace") assert "NOT_JSON_LINE" in out @@ -182,6 +216,8 @@ def _proxy_records_tools_call_and_exit_code(tmp_path): assert call_rows[1]["is_error"] is True assert meta["unparsed_lines"] >= 1 assert meta["relayed_lines"] >= 3 + assert meta["finalization_reason"] in {"normal_eof", "child_exit"} + assert meta["finalization_signal"] is None def _proxy_byte_faithful_child_receives_exact_bytes(tmp_path): @@ -561,6 +597,96 @@ def test_proxy_behaviours(case, tmp_path): case(tmp_path) +@pytest.mark.parametrize( + "signum", + [ + pytest.param(signal.SIGTERM, id="SIGTERM"), + pytest.param(signal.SIGHUP, id="SIGHUP"), + pytest.param(signal.SIGINT, id="SIGINT"), + ], +) +def test_proxy_signal_finalization_writes_meta_and_preserves_signal_exit(tmp_path: Path, signum: int): + server = _write_fake_server(tmp_path / "signal_server.py") + sidecar = tmp_path / "signal-sidecar.jsonl" + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + assert proc.stdin is not None + messages = [ + _request(1, "tools/list"), + _request(2, "tools/call", {"name": "list_work_items", "arguments": {"project": "P"}}), + ] + proc.stdin.write(("\n".join(json.dumps(message) for message in messages) + "\n").encode("utf-8")) + proc.stdin.flush() + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + calls, _status = load_proxy_sidecar(sidecar) + if len(calls) == 1: + break + time.sleep(0.01) + else: + pytest.fail("proxy did not record the completed tool call before signal") + + os.kill(proc.pid, signum) + returncode = proc.wait(timeout=SHUTDOWN_DEADLINE_S + 5.0) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=2.0) + if proc.stdin is not None: + proc.stdin.close() + + rows = [json.loads(line) for line in sidecar.read_text(encoding="utf-8").splitlines() if line.strip()] + assert rows[-1]["row_type"] == "proxy_meta" + meta = rows[-1] + assert meta["finalization_reason"] == "signal" + assert meta["finalization_signal"] == signal.Signals(signum).name + assert returncode == -signum + + # SIGTERM/SIGHUP use the controlled drain. A completed request remains a + # complete trace; a signal is diagnostic, not intrinsically recorder loss. + if signum in (signal.SIGTERM, signal.SIGHUP): + _calls, status = load_proxy_sidecar(sidecar) + assert status["state"] == "complete" + assert status["pending_left"] == 0 + assert status["pumps_alive"] is False + assert status["tool_manifest_fingerprint"] + + +def test_signal_finalization_does_not_hide_pending_tool_loss(tmp_path: Path): + sidecar = tmp_path / "signal-pending.jsonl" + recorder = SidecarRecorder(sidecar) + recorder.finalization_reason = "signal" + recorder.finalization_signal = "SIGTERM" + recorder.on_client_message(_request(1, "tools/call", {"name": "unfinished", "arguments": {}})) + recorder.write_meta() + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], sidecar, notes) + + assert result.status["finalization_reason"] == "signal" + assert result.status["finalization_signal"] == "SIGTERM" + assert result.status["pending_left"] == 1 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert any(note.startswith("proxy_sidecar_incomplete:pending_left=1") for note in notes) + + def _sidecar_recorder_unit(tmp_path): sentinel = "hidden-target-fact-7b0a1f9c" rec = SidecarRecorder( @@ -941,7 +1067,7 @@ def test_scrub_child_pythonpath_removes_repo(): assert "PYTHONPATH" not in only -def test_proxy_consumes_private_evidence_file_before_starting_mcp(tmp_path: Path, monkeypatch): +def test_proxy_loads_reusable_private_evidence_file_before_starting_mcp(tmp_path: Path, monkeypatch): sentinel = "hidden-target-fact-7b0a1f9c" evidence_file = tmp_path / "evidence.json" write_evidence_config( @@ -954,7 +1080,7 @@ def test_proxy_consumes_private_evidence_file_before_starting_mcp(tmp_path: Path captured = {} def fake_run_proxy(command, log_path, **kwargs): - assert not evidence_file.exists(), "proxy must unlink evidence before starting Plane MCP" + assert evidence_file.is_file(), "run-scoped evidence must remain for later proxy sessions" captured.update({"command": command, "log_path": log_path, **kwargs}) return 0 @@ -972,11 +1098,104 @@ def fake_run_proxy(command, log_path, **kwargs): ) assert rc == 0 + assert evidence_file.is_file() + assert sentinel not in evidence_file.read_text(encoding="utf-8") assert set(captured["evidence_fingerprints"]) == {TARGET_ENTITY_EVIDENCE} assert captured["evidence_targets"] == {TARGET_ENTITY_EVIDENCE: ("target-1",)} assert captured["evidence_aggregates"] == {} +def test_probe_then_session_reuses_evidence_config_and_records_provenance(tmp_path: Path): + sentinel = "hidden-target-fact-7b0a1f9c" + evidence_file = tmp_path / "evidence.json" + write_evidence_config( + evidence_file, + {TARGET_ENTITY_EVIDENCE: [sentinel]}, + {TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 + assert sentinel not in evidence_file.read_text(encoding="utf-8") + + server = tmp_path / "evidence_server.py" + server.write_text( + textwrap.dedent( + f""" + import json, sys + sentinel = {sentinel!r} + for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + if method == "tools/list": + result = {{"tools": [{{"name": "read_target", "inputSchema": {{"type": "object"}}}}]}} + elif method == "tools/call": + result = {{ + "content": [{{"type": "text", "text": f"target={{sentinel}}"}}], + "isError": False, + }} + else: + result = {{}} + sys.stdout.write(json.dumps({{"jsonrpc": "2.0", "id": message["id"], "result": result}}) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + + def run_session(sidecar: Path, request: dict) -> tuple[list[dict], dict]: + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--evidence-file", + str(evidence_file), + "--", + sys.executable, + str(server), + ], + input=(json.dumps(request) + "\n").encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode("utf-8", errors="replace") + return load_proxy_sidecar(sidecar) + + probe_calls, probe_status = run_session(tmp_path / "probe.jsonl", _request(1, "tools/list")) + assert probe_calls == [] + assert probe_status["state"] == "complete" + assert probe_status["meta"]["evidence_trace_available"] is True + assert evidence_file.is_file() + assert sentinel not in evidence_file.read_text(encoding="utf-8") + + session_calls, session_status = run_session( + tmp_path / "session.jsonl", + _request( + 2, + "tools/call", + {"name": "read_target", "arguments": {"work_item_id": "target-1"}}, + ), + ) + assert session_status["state"] == "complete" + assert session_status["meta"]["evidence_trace_available"] is True + assert len(session_calls) == 1 + assert session_calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert evidence_file.is_file() + assert sentinel not in evidence_file.read_text(encoding="utf-8") + + +def test_missing_or_malformed_evidence_config_fails_closed(tmp_path: Path): + empty = ({}, {}, {}) + assert consume_evidence_config(tmp_path / "missing.json") == empty + + malformed = tmp_path / "malformed.json" + malformed.write_text("{not-json", encoding="utf-8") + assert consume_evidence_config(malformed) == empty + assert malformed.is_file() + + def test_rapid_response_pairing(tmp_path: Path): """Record-before-forward: fast child responses must pair with requests (no unmatched). @@ -1324,28 +1543,95 @@ def test_sidecar_rejects_invalid_duplicate_and_gapped_sequences( assert f"{status_key}={status[status_key]}" in notes[0] -@pytest.mark.parametrize("case", ["duplicate", "not-final"]) -def test_sidecar_requires_exactly_one_final_proxy_meta(tmp_path: Path, case: str): - path = tmp_path / f"meta-{case}.jsonl" - meta = {"row_type": "proxy_meta", "pending_left": 0, "last_seq": 0, "tool_request_count": 0} - if case == "duplicate": - rows = [meta, meta] - else: - meta.update({"last_seq": 1, "tool_request_count": 1}) - rows = [meta, {"tool": "late", "args": {}, "seq": 1}] +def test_zero_call_probe_then_real_session_is_complete(tmp_path: Path): + path = tmp_path / "probe-then-real.jsonl" + manifest = "31c209e40544" + rows = [ + _complete_proxy_meta(0, manifest), + _proxy_call("list_projects", 1), + _proxy_call("search_work_items", 2), + _proxy_call("create_work_log", 3), + _complete_proxy_meta(3, manifest), + ] path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") - _, status = load_proxy_sidecar(path) notes: list[str] = [] - apply_proxy_sidecar([], [], path, notes) + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "complete" + assert result.status["session_count"] == 2 + assert result.status["proxy_meta_count"] == 2 + assert result.status["duplicate_seq"] == 0 + assert [call["tool"] for call in result.calls] == [ + "list_projects", + "search_work_items", + "create_work_log", + ] + assert result.trace_integrity is True + assert result.tool_manifest_fingerprint == manifest + assert not any("incomplete" in note for note in notes) - assert status["state"] == "incomplete" - if case == "duplicate": - assert status["proxy_meta_count"] == 2 - assert "proxy_meta_count=2" in notes[0] - else: - assert status["proxy_meta_not_final"] is True - assert "proxy_meta_not_final=1" in notes[0] + +def test_two_calling_sessions_validate_sequences_independently(tmp_path: Path): + path = tmp_path / "two-calling-sessions.jsonl" + rows = [ + _proxy_call("session-1-call-2", 2), + _proxy_call("session-1-call-1", 1), + _complete_proxy_meta(2), + _proxy_call("session-2-call-3", 3), + _proxy_call("session-2-call-1", 1), + _proxy_call("session-2-call-2", 2), + _complete_proxy_meta(3), + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + calls, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["duplicate_seq"] == 0 + assert status["missing_seq"] == 0 + assert status["unexpected_seq"] == 0 + assert [call["tool"] for call in calls] == [ + "session-1-call-1", + "session-1-call-2", + "session-2-call-1", + "session-2-call-2", + "session-2-call-3", + ] + assert [call["seq"] for call in calls] == [1, 2, 1, 2, 3] + + +def test_trailing_unfinalized_proxy_session_stays_fatal(tmp_path: Path): + path = tmp_path / "trailing-unfinalized.jsonl" + rows = [_complete_proxy_meta(0), _proxy_call("late", 1)] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "incomplete" + assert result.status["proxy_meta_count"] == 1 + assert result.status["unfinalized_sessions"] == 1 + assert result.status["proxy_meta_not_final"] is True + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert any("unfinalized_sessions=1" in note for note in notes) + + +def test_disagreeing_session_manifests_are_reported_and_invalidated(tmp_path: Path): + path = tmp_path / "manifest-disagreement.jsonl" + rows = [_complete_proxy_meta(0, "manifest-a"), _complete_proxy_meta(0, "manifest-b")] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "complete" + assert result.status["tool_manifest_disagreement"] is True + assert result.status["tool_manifest_fingerprints"] == ["manifest-a", "manifest-b"] + assert result.trace_integrity is True + assert result.tool_manifest_fingerprint is None + assert "proxy_tool_manifest_disagreement:manifest-a,manifest-b" in notes @pytest.mark.parametrize("case", ["missing-last-seq", "request-count-mismatch"]) From 619e9370a0f54fb8a6ceb5b3aa0cb42cc780667b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 08:59:32 +0530 Subject: [PATCH 43/93] Stop leaking the answers, and give each proxy its own file Five independent graders audited the harness; four found the first defect below and three found the second. Both came from composing changes that were each defensible alone. The evidence configuration serialized raw aggregate truth: total_count carried R2's urgent-open count and grouped_counts carried R6's per-project bug counts, in a file the agent's own MCP config points at and which had stopped being unlinked. An agent could read the answers instead of querying Plane. Hashing a small integer is not protection, so the responsibility is inverted: the configuration now carries only which aggregate shape to extract and which entity to target, the proxy records what it observed, and the verifier compares that against seed truth it already holds after the agent exits. Grouped counts are bound to the request target the same way total counts already were. Every proxy truncated the shared sidecar on construction, so the second session the CLI starts per task erased the first while leaving a structurally valid final metadata row - loss the loader reported as complete. Each proxy now owns its own file and the loader merges the sessions it finds, which removes the truncation and the interleaving hazard together. The tests that appeared to cover this used two different files or hand-built one; the new one launches two real proxies. Absent trace integrity is no longer read as true, mixed present and missing manifests are a mismatch rather than a silent pass, and the live headline now uses the task-cluster interval the reporter already used. Agents also saw every globally configured MCP server, including a Node REPL that can reach Plane directly. Each CLI now runs against an isolated configuration home, verified by reading back the effective server list rather than by trusting the flag. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/__init__.py | 8 + evals/drivers/cli/antigravity.py | 12 +- evals/drivers/cli/codex.py | 60 +++++- evals/drivers/cli/opencode.py | 38 +++- evals/drivers/cli/sidecar.py | 253 ++++++++++++++----------- evals/drivers/driver.py | 40 ++-- evals/evidence.py | 101 +++++++--- evals/proxy.py | 50 ++--- evals/report/identity.py | 7 +- evals/report/summary.py | 6 +- evals/results.py | 8 +- evals/runner/live.py | 11 +- tests/evals/drivers/test_api_driver.py | 10 +- tests/evals/drivers/test_cli_driver.py | 190 ++++++++++++++++++- tests/evals/report/test_compare.py | 26 +-- tests/evals/report/test_identity.py | 16 +- tests/evals/report/test_load.py | 12 +- tests/evals/report/test_summary.py | 53 +++++- tests/evals/report/test_table.py | 4 + tests/evals/runner/test_live.py | 33 +++- tests/evals/test_proxy.py | 190 ++++++++++++++++--- uv.lock | 152 ++++++++++++++- 22 files changed, 1019 insertions(+), 261 deletions(-) diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 686a8d67..c5738996 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -26,10 +26,13 @@ find_codex_rollout, parse_codex_jsonl_events, parse_codex_rollout_calls, + prepare_codex_home, + write_codex_mcp_config, write_codex_mcp_override_args, ) from evals.drivers.cli.opencode import ( OpencodeCliDriver, + prepare_opencode_isolated_environment, write_opencode_mcp_config, ) from evals.drivers.cli.process import kill_process_group, note_timeout_kill, run_cli_subprocess @@ -40,6 +43,7 @@ load_proxy_sidecar, load_proxy_sidecar_calls, proxy_pid_path, + proxy_session_paths, proxy_wrap_server_command, wait_for_proxy_meta, ) @@ -92,12 +96,16 @@ def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: "parse_codex_jsonl_events", "parse_codex_rollout_calls", "prepare_antigravity_fake_home", + "prepare_codex_home", + "prepare_opencode_isolated_environment", "proxy_pid_path", + "proxy_session_paths", "proxy_wrap_server_command", "run_cli_subprocess", "wait_for_proxy_meta", "write_antigravity_mcp_config", "write_claude_mcp_config", "write_codex_mcp_override_args", + "write_codex_mcp_config", "write_opencode_mcp_config", ] diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index e6ace9ec..79528129 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -137,7 +137,17 @@ def write_mcp_config( args=server_command[1:], env=child_env, ) - run_env = {**os.environ, "HOME": str(fake_home)} + xdg_roots = { + name: temp_dir / name.lower().replace("_home", "") + for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") + } + for directory in xdg_roots.values(): + directory.mkdir(parents=True, exist_ok=True) + run_env = { + **os.environ, + "HOME": str(fake_home), + **{name: str(directory) for name, directory in xdg_roots.items()}, + } if "PATH" in child_env: run_env["PATH"] = child_env["PATH"] return CliLaunch(cwd=task_cwd, env=run_env) diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index 4858b596..3b2038de 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -9,6 +9,8 @@ from __future__ import annotations import json +import os +import shutil import subprocess from collections.abc import Callable from pathlib import Path @@ -222,6 +224,51 @@ def write_codex_mcp_override_args( return out +def write_codex_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write the complete MCP config for an isolated Codex home.""" + lines = [ + f"[mcp_servers.{json.dumps(server_name)}]", + f"command = {json.dumps(command)}", + f"args = {json.dumps(args)}", + ] + if env: + lines.append(f"[mcp_servers.{json.dumps(server_name)}.env]") + lines.extend(f"{json.dumps(key)} = {json.dumps(value)}" for key, value in env.items()) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def prepare_codex_home( + codex_home: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + real_codex_home: Path | None = None, +) -> None: + """Create an exclusive config root while copying only the CLI login artifact.""" + write_codex_mcp_config( + codex_home / "config.toml", + command=command, + args=args, + env=env, + ) + source_home = real_codex_home or Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") + source_auth = source_home / "auth.json" + if source_auth.is_file(): + try: + shutil.copy2(source_auth, codex_home / "auth.json") + except OSError: + pass + + # Codex CLI driver (experimental — do not spend live quota from CI) # --------------------------------------------------------------------------- @@ -277,14 +324,17 @@ def write_mcp_config( server_command: list[str], child_env: dict[str, str], ) -> CliLaunch: - del temp_dir - config_args = write_codex_mcp_override_args( + codex_home = temp_dir / "codex-home" + prepare_codex_home( + codex_home, command=server_command[0], args=server_command[1:], env=child_env, - server_name="plane", ) - return CliLaunch(cwd=task_cwd, config_args=config_args) + run_env = {**os.environ, "CODEX_HOME": str(codex_home)} + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + return CliLaunch(cwd=task_cwd, env=run_env) def build_command( self, @@ -381,5 +431,7 @@ def parse_output( "find_codex_rollout", "parse_codex_jsonl_events", "parse_codex_rollout_calls", + "prepare_codex_home", + "write_codex_mcp_config", "write_codex_mcp_override_args", ] diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py index 2816a26b..db6abca9 100644 --- a/evals/drivers/cli/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import os +import shutil import subprocess from collections.abc import Callable from pathlib import Path @@ -40,6 +42,36 @@ def write_opencode_mcp_config( path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") +def prepare_opencode_isolated_environment(temp_dir: Path) -> dict[str, str]: + """Return an environment whose HOME/XDG roots cannot load user MCP config.""" + fake_home = temp_dir / "home" + xdg_config = temp_dir / "xdg-config" + xdg_data = temp_dir / "xdg-data" + xdg_cache = temp_dir / "xdg-cache" + xdg_state = temp_dir / "xdg-state" + for directory in (fake_home, xdg_config, xdg_data, xdg_cache, xdg_state): + directory.mkdir(parents=True, exist_ok=True) + + real_data_root = Path(os.environ.get("XDG_DATA_HOME") or Path.home() / ".local" / "share") + source_auth = real_data_root / "opencode" / "auth.json" + if source_auth.is_file(): + destination = xdg_data / "opencode" / "auth.json" + destination.parent.mkdir(parents=True, exist_ok=True) + try: + shutil.copy2(source_auth, destination) + except OSError: + pass + + return { + **os.environ, + "HOME": str(fake_home), + "XDG_CONFIG_HOME": str(xdg_config), + "XDG_DATA_HOME": str(xdg_data), + "XDG_CACHE_HOME": str(xdg_cache), + "XDG_STATE_HOME": str(xdg_state), + } + + class OpencodeCliDriver(CliDriver): """Run tasks via ``opencode run`` (proxy-first call recording). @@ -90,7 +122,10 @@ def write_mcp_config( env=child_env, server_name="plane", ) - return CliLaunch(cwd=temp_dir) + run_env = prepare_opencode_isolated_environment(temp_dir) + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + return CliLaunch(cwd=temp_dir, env=run_env) def build_command( self, @@ -148,5 +183,6 @@ def parse_output( __all__ = [ "OpencodeCliDriver", + "prepare_opencode_isolated_environment", "write_opencode_mcp_config", ] diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 4dc3e2e1..15e7f650 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -75,21 +75,16 @@ def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: return out -def load_proxy_sidecar( - path: Path, -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Load and validate the sidecar as independently finalized proxy sessions. - - Status keys: - - missing / empty / complete / incomplete - - torn_line: final line failed to parse - - skipped_rows: non-final rows that could not produce a call or metadata row - - meta / metas / sessions: final metadata plus per-session validation - - pending_left / non_tool_pending_left: sums across sessions - - sequence errors: sums of per-session invalid/duplicate/missing/unexpected values - - proxy_meta_not_final: a trailing session lacks its terminating metadata - - tool_manifest_disagreement: finalized sessions advertised different surfaces - """ +def proxy_session_paths(path: Path) -> list[Path]: + """Discover the legacy base file and every per-process session derived from it.""" + discovered = list(path.parent.glob(f"{path.name}.*.jsonl")) + if path.is_file(): + discovered.append(path) + return sorted(set(discovered), key=lambda item: item.name) + + +def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Load, validate, and merge exactly one proxy session from each sidecar file.""" status: dict[str, Any] = { "state": "missing", "torn_line": False, @@ -98,6 +93,9 @@ def load_proxy_sidecar( "metas": [], "sessions": [], "session_count": 0, + "session_file_count": 0, + "session_files": [], + "all_sessions_finalized": False, "unfinalized_sessions": 0, "pending_left": None, "non_tool_pending_left": None, @@ -112,71 +110,77 @@ def load_proxy_sidecar( "tool_manifest_fingerprints": [], "tool_manifest_missing_sessions": 0, } - if not path.is_file(): - return [], status - try: - raw = path.read_bytes() - except OSError: - return [], status - if not raw: - status["state"] = "empty" + session_paths = proxy_session_paths(path) + if not session_paths: return [], status - - # Each proxy process restarts seq at 1 and terminates its segment with - # proxy_meta. Validate and order calls within those boundaries, never over - # the whole file where healthy sessions can have colliding seq values. - lines = raw.decode("utf-8", errors="replace").splitlines() - raw_segments: list[dict[str, Any]] = [] - current: dict[str, Any] = {"calls": [], "meta": None, "torn_line": False, "skipped_rows": 0} - current_has_rows = False - for i, line in enumerate(lines): - s = line.strip() - if not s: - continue - current_has_rows = True + status["session_file_count"] = len(session_paths) + status["session_files"] = [str(session_path) for session_path in session_paths] + + raw_sessions: list[dict[str, Any]] = [] + for session_path in session_paths: + raw_session: dict[str, Any] = { + "path": session_path, + "calls": [], + "metas": [], + "torn_line": False, + "skipped_rows": 0, + "proxy_meta_not_final": False, + } try: - row = json.loads(s) - except json.JSONDecodeError: - # Tolerate a torn final line (crash mid-write); stop there. - if i == len(lines) - 1: - current["torn_line"] = True - break - current["skipped_rows"] += 1 - continue - if not isinstance(row, dict): - current["skipped_rows"] += 1 + raw = session_path.read_bytes() + except OSError: + raw_sessions.append(raw_session) continue - if row.get("row_type") == "proxy_meta": - current["meta"] = row - raw_segments.append(current) - current = {"calls": [], "meta": None, "torn_line": False, "skipped_rows": 0} - current_has_rows = False - continue - tool = row.get("tool") - if not tool: - current["skipped_rows"] += 1 - continue - call = { - "tool": str(tool), - "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), - "origin": "plane", - "is_error": bool(row.get("is_error")), - "result_chars": int(row.get("result_chars") or 0), - "duration_ms": row.get("duration_ms"), - "seq": row.get("seq"), - } - # Optional in new sidecars; old payload-free rows remain valid. - if isinstance(row.get("result_text"), str): - call["result_text"] = row["result_text"] - if isinstance(row.get("observed_sentinels"), list): - call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] - current["calls"].append(call) - - if current_has_rows: - raw_segments.append(current) - if not raw_segments: - status["state"] = "empty" - return [], status + lines = raw.decode("utf-8", errors="replace").splitlines() + last_row_kind: str | None = None + for index, line in enumerate(lines): + text = line.strip() + if not text: + continue + try: + row = json.loads(text) + except json.JSONDecodeError: + if index == len(lines) - 1: + raw_session["torn_line"] = True + raw_session["proxy_meta_not_final"] = bool(raw_session["metas"]) + break + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + if not isinstance(row, dict): + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + if row.get("row_type") == "proxy_meta": + raw_session["metas"].append(row) + last_row_kind = "meta" + continue + tool = row.get("tool") + if not tool: + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + call = { + "tool": str(tool), + "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), + "origin": "plane", + "is_error": bool(row.get("is_error")), + "result_chars": int(row.get("result_chars") or 0), + "duration_ms": row.get("duration_ms"), + "seq": row.get("seq"), + } + if isinstance(row.get("result_text"), str): + call["result_text"] = row["result_text"] + if isinstance(row.get("observed_sentinels"), list): + call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] + if isinstance(row.get("observed_aggregates"), list): + call["observed_aggregates"] = [value for value in row["observed_aggregates"] if isinstance(value, dict)] + raw_session["calls"].append(call) + last_row_kind = "call" + raw_session["proxy_meta_not_final"] = raw_session["proxy_meta_not_final"] or bool( + raw_session["metas"] and last_row_kind != "meta" + ) + raw_sessions.append(raw_session) counter_keys = ( "pending_left", @@ -202,19 +206,24 @@ def load_proxy_sidecar( calls: list[dict[str, Any]] = [] session_statuses: list[dict[str, Any]] = [] manifests: list[str] = [] - for index, raw_segment in enumerate(raw_segments): - segment_calls = raw_segment["calls"] - meta = raw_segment["meta"] - valid_sequences = [call["seq"] for call in segment_calls if _nonnegative_int(call.get("seq")) not in (None, 0)] + for index, raw_session in enumerate(raw_sessions): + session_calls = raw_session["calls"] + meta_rows = raw_session["metas"] + meta = meta_rows[-1] if meta_rows else None + valid_sequences = [call["seq"] for call in session_calls if _nonnegative_int(call.get("seq")) not in (None, 0)] last_seq = _nonnegative_int(meta.get("last_seq")) if meta is not None else None tool_request_count = _nonnegative_int(meta.get("tool_request_count")) if meta is not None else None segment: dict[str, Any] = { "index": index, + "path": str(raw_session["path"]), "meta": meta, - "call_count": len(segment_calls), - "torn_line": bool(raw_segment["torn_line"]), - "skipped_rows": int(raw_segment["skipped_rows"]), - "invalid_seq": len(segment_calls) - len(valid_sequences), + "meta_count": len(meta_rows), + "call_count": len(session_calls), + "torn_line": bool(raw_session["torn_line"]), + "skipped_rows": int(raw_session["skipped_rows"]), + "proxy_meta_not_final": bool(raw_session["proxy_meta_not_final"]), + "finalized": len(meta_rows) == 1 and not raw_session["proxy_meta_not_final"], + "invalid_seq": len(session_calls) - len(valid_sequences), "duplicate_seq": len(valid_sequences) - len(set(valid_sequences)), "missing_seq": 0, "unexpected_seq": 0, @@ -224,7 +233,9 @@ def load_proxy_sidecar( "tool_request_count": tool_request_count, } if meta is not None: - segment["invalid_meta_fields"] = int(last_seq is None) + int(tool_request_count is None) + segment["invalid_meta_fields"] = ( + abs(len(meta_rows) - 1) + int(last_seq is None) + int(tool_request_count is None) + ) if last_seq is not None and tool_request_count is not None and tool_request_count != last_seq: segment["invalid_meta_fields"] += 1 expected_sequences = set(range(1, last_seq + 1)) if last_seq is not None else set() @@ -246,32 +257,32 @@ def load_proxy_sidecar( segment["state"] = ( "incomplete" if meta is None + or not segment["finalized"] or segment["torn_line"] or segment["skipped_rows"] > 0 or any((segment.get(key) or 0) > 0 for key in fatal_counts) or segment["pumps_alive"] else "complete" ) - # Preserve request order inside a session, then concatenate sessions in - # file order. Never globally sort colliding per-process seq values. - segment_calls.sort( + session_calls.sort( key=lambda call: ( _nonnegative_int(call.get("seq")) in (None, 0), _nonnegative_int(call.get("seq")) or 0, ) ) - calls.extend(segment_calls) + calls.extend(session_calls) session_statuses.append(segment) metas = [segment["meta"] for segment in session_statuses if segment["meta"] is not None] - unfinalized_sessions = sum(segment["meta"] is None for segment in session_statuses) + unfinalized_sessions = sum(not segment["finalized"] for segment in session_statuses) status["sessions"] = session_statuses status["session_count"] = len(session_statuses) status["metas"] = metas status["meta"] = metas[-1] if metas else None - status["proxy_meta_count"] = len(metas) + status["proxy_meta_count"] = sum(segment["meta_count"] for segment in session_statuses) status["unfinalized_sessions"] = unfinalized_sessions - status["proxy_meta_not_final"] = unfinalized_sessions > 0 + status["all_sessions_finalized"] = bool(session_statuses) and unfinalized_sessions == 0 + status["proxy_meta_not_final"] = any(segment["proxy_meta_not_final"] for segment in session_statuses) status["torn_line"] = any(segment["torn_line"] for segment in session_statuses) status["pumps_alive"] = any(segment["pumps_alive"] for segment in session_statuses) aggregate_keys = ( @@ -294,12 +305,22 @@ def load_proxy_sidecar( status["tool_manifest_fingerprint"] = ( unique_manifests[0] if len(unique_manifests) == 1 and missing_manifests == 0 else None ) + status["evidence_trace_available"] = ( + bool(metas) + and len(metas) == len(session_statuses) + and all(bool(meta.get("evidence_trace_available")) for meta in metas) + ) if status["meta"] is not None: status["finalization_reason"] = status["meta"].get("finalization_reason") status["finalization_signal"] = status["meta"].get("finalization_signal") - status["state"] = ( - "incomplete" if any(segment["state"] == "incomplete" for segment in session_statuses) else "complete" - ) + if all(segment["call_count"] == 0 and segment["meta_count"] == 0 for segment in session_statuses) and not ( + status["torn_line"] or status["skipped_rows"] + ): + status["state"] = "empty" + else: + status["state"] = ( + "incomplete" if any(segment["state"] == "incomplete" for segment in session_statuses) else "complete" + ) return calls, status @@ -360,12 +381,23 @@ def proxy_pid_path(sidecar_path: Path) -> Path: def _read_proxy_pid(sidecar_path: Path) -> int | None: - try: - raw = proxy_pid_path(sidecar_path).read_text(encoding="ascii").strip() - pid = int(raw) - except (OSError, UnicodeError, ValueError): - return None - return pid if pid > 0 else None + pids = _read_proxy_pids(sidecar_path) + return pids[-1] if pids else None + + +def _read_proxy_pids(sidecar_path: Path) -> list[int]: + lifecycle_paths = [proxy_pid_path(path) for path in proxy_session_paths(sidecar_path)] + lifecycle_paths.extend(sidecar_path.parent.glob(f"{sidecar_path.name}.*.jsonl.pid")) + lifecycle_paths.append(proxy_pid_path(sidecar_path)) + pids: set[int] = set() + for lifecycle_path in set(lifecycle_paths): + try: + pid = int(lifecycle_path.read_text(encoding="ascii").strip()) + except (OSError, UnicodeError, ValueError): + continue + if pid > 0: + pids.add(pid) + return sorted(pids) def _process_is_alive(pid: int) -> bool: @@ -396,17 +428,17 @@ def _wait_for_proxy_meta_outcome( # A completed CLI cannot launch a proxy after the fact. No trace and no # lifecycle file therefore means the proxy was never observed, rather than # a finalizer that could benefit from waiting for the whole deadline. - pid = _read_proxy_pid(sidecar_path) - if not sidecar_path.exists() and pid is None: + pids = _read_proxy_pids(sidecar_path) + if not proxy_session_paths(sidecar_path) and not pids: return "proxy_not_observed" deadline = time.monotonic() + max(0.0, max_wait_s) while True: _, status = load_proxy_sidecar(sidecar_path) - if status.get("meta") is not None: + if status.get("all_sessions_finalized"): return "meta_present" - pid = _read_proxy_pid(sidecar_path) - if pid is not None and not _process_is_alive(pid): + pids = _read_proxy_pids(sidecar_path) + if pids and not any(_process_is_alive(pid) for pid in pids): return "proxy_exited" rem = deadline - time.monotonic() if rem <= 0: @@ -416,10 +448,10 @@ def _wait_for_proxy_meta_outcome( # Close the boundary races in both directions: metadata may have landed on # the final sleep, or the proxy may have exited without writing it. _, status = load_proxy_sidecar(sidecar_path) - if status.get("meta") is not None: + if status.get("all_sessions_finalized"): return "meta_present" - pid = _read_proxy_pid(sidecar_path) - if pid is not None and not _process_is_alive(pid): + pids = _read_proxy_pids(sidecar_path) + if pids and not any(_process_is_alive(pid) for pid in pids): return "proxy_exited" return "timeout" @@ -551,6 +583,7 @@ def harvest_proxy_after_cli_timeout( "load_proxy_sidecar", "load_proxy_sidecar_calls", "proxy_pid_path", + "proxy_session_paths", "proxy_wrap_server_command", "ProxySidecarResult", "trace_integrity_from_status", diff --git a/evals/drivers/driver.py b/evals/drivers/driver.py index 85418e64..9e53a55a 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/driver.py @@ -43,6 +43,7 @@ normalize_evidence_sentinels, normalize_evidence_targets, observed_aggregate_labels, + observed_aggregates, observed_sentinel_labels, write_evidence_config, ) @@ -358,6 +359,13 @@ async def _run_task( calls[idx]["is_error"] = result.is_error calls[idx]["duration_ms"] = duration_ms if evidence_active: + aggregate_observations = observed_aggregates( + result.text, + aggregates, + request_args=calls[idx]["args"], + evidence_targets=targets, + ) + calls[idx]["observed_aggregates"] = aggregate_observations calls[idx]["observed_sentinels"] = sorted( set( observed_sentinel_labels( @@ -367,14 +375,7 @@ async def _run_task( evidence_targets=targets, ) ) - | set( - observed_aggregate_labels( - result.text, - aggregates, - request_args=calls[idx]["args"], - evidence_targets=targets, - ) - ) + | set(observed_aggregate_labels(aggregate_observations, aggregates)) ) pending_results.append((idx, result.text)) if matched_ids != set(call_indices) or len(call_indices) != len(turn.tool_calls): @@ -619,6 +620,15 @@ def run_task( targets = normalize_evidence_targets(evidence_targets) aggregates = normalize_evidence_aggregates(evidence_aggregates) evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) + + def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: + """Turn observed proxy values into labels using harness-held seed truth.""" + for call in calls: + labels = set(call.get("observed_sentinels") or []) + labels.update(observed_aggregate_labels(call.get("observed_aggregates"), aggregates)) + if evidence_active: + call["observed_sentinels"] = sorted(labels) + real_command = ( list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] ) @@ -674,17 +684,15 @@ def run_task( notes, ) calls, client_calls, call_source = sidecar_result + verify_aggregate_observations(calls) trace_integrity = sidecar_result.trace_integrity trace_integrity_reason = sidecar_result.trace_integrity_reason tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint evidence_available = False if evidence_active and call_source == "proxy": _proxy_calls, status = load_proxy_sidecar(sidecar) - meta = status.get("meta") if isinstance(status, dict) else None evidence_available = bool( - status.get("state") == "complete" - and isinstance(meta, dict) - and meta.get("evidence_trace_available") + status.get("state") == "complete" and status.get("evidence_trace_available") ) if not evidence_available: notes.append("proxy_response_evidence_unavailable") @@ -735,6 +743,7 @@ def run_task( calls, client_calls, proxy_source = sidecar_result output.calls = calls output.client_tool_calls = client_calls + verify_aggregate_observations(output.calls) trace_integrity = sidecar_result.trace_integrity trace_integrity_reason = sidecar_result.trace_integrity_reason tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint @@ -744,12 +753,7 @@ def run_task( evidence_available = False if evidence_active and output.call_source == "proxy": _proxy_calls, status = load_proxy_sidecar(sidecar) - meta = status.get("meta") if isinstance(status, dict) else None - evidence_available = bool( - status.get("state") == "complete" - and isinstance(meta, dict) - and meta.get("evidence_trace_available") - ) + evidence_available = bool(status.get("state") == "complete" and status.get("evidence_trace_available")) if not evidence_available: notes.append("proxy_response_evidence_unavailable") diff --git a/evals/evidence.py b/evals/evidence.py index c2b56df4..263d0be1 100644 --- a/evals/evidence.py +++ b/evals/evidence.py @@ -85,6 +85,35 @@ def normalize_evidence_aggregates(value: Any) -> dict[str, tuple[dict[str, Any], return normalized +def evidence_aggregate_shapes(value: Any) -> dict[str, tuple[dict[str, str], ...]]: + """Reduce aggregate truth to the response shapes safe for an agent-visible proxy.""" + return { + label: tuple({"kind": str(spec["kind"])} for spec in specs) + for label, specs in normalize_evidence_aggregates(value).items() + } + + +def normalize_evidence_aggregate_shapes(value: Any) -> dict[str, tuple[dict[str, str], ...]]: + """Validate aggregate extraction instructions that contain no expected values.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[dict[str, str], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[dict[str, str]] = [] + for raw_spec in raw_specs: + if not isinstance(raw_spec, Mapping) or raw_spec.get("kind") not in {"total_count", "grouped_counts"}: + continue + spec = {"kind": str(raw_spec["kind"])} + if spec not in specs: + specs.append(spec) + if specs: + normalized[label] = tuple(specs) + return normalized + + def configured_evidence_labels(sentinels: Any, targets: Any, aggregates: Any = None) -> tuple[str, ...]: """Return labels that have both response values and target entity IDs.""" values_by_label = normalize_evidence_sentinels(sentinels) @@ -154,7 +183,7 @@ def decode_evidence_sentinels(value: str | None) -> dict[str, tuple[str, ...]]: def encode_evidence_config(sentinels: Any, targets: Any, aggregates: Any = None) -> str: - """Serialize targets plus one-way value fingerprints, never raw sentinels.""" + """Serialize targets and extraction shapes, never raw sentinels or aggregate truth.""" fingerprints = fingerprint_evidence_sentinels(sentinels) return json.dumps( { @@ -163,7 +192,7 @@ def encode_evidence_config(sentinels: Any, targets: Any, aggregates: Any = None) for label, specs in fingerprints.items() }, "targets": normalize_evidence_targets(targets), - "aggregates": normalize_evidence_aggregates(aggregates), + "aggregates": evidence_aggregate_shapes(aggregates), }, ensure_ascii=True, separators=(",", ":"), @@ -189,7 +218,7 @@ def decode_evidence_config( return ( normalize_evidence_fingerprints(raw.get("fingerprints")), normalize_evidence_targets(raw.get("targets")), - normalize_evidence_aggregates(raw.get("aggregates")), + normalize_evidence_aggregate_shapes(raw.get("aggregates")), ) @@ -329,42 +358,67 @@ def _decoded_documents(response_text: str) -> list[Any]: return documents -def observed_aggregate_labels( +def observed_aggregates( response_text: str, aggregates: Any, *, request_args: Any, evidence_targets: Any, -) -> list[str]: - """Match exact aggregate result fields while retaining seeded target binding.""" - specs_by_label = normalize_evidence_aggregates(aggregates) +) -> list[dict[str, Any]]: + """Extract target-bound aggregate values without receiving expected truth.""" + specs_by_label = normalize_evidence_aggregate_shapes(aggregates) targets_by_label = normalize_evidence_targets(evidence_targets) documents = _decoded_documents(response_text) - matched: list[str] = [] + observations: list[dict[str, Any]] = [] for label, specs in specs_by_label.items(): targets = targets_by_label.get(label, ()) + if not targets or not _request_targets(request_args, targets): + continue for spec in specs: if spec["kind"] == "total_count": - if not _request_targets(request_args, targets): - continue - if any(isinstance(value, Mapping) and value.get("total_count") == spec["value"] for value in documents): - matched.append(label) - break + for value in documents: + if not isinstance(value, Mapping): + continue + count = value.get("total_count") + if isinstance(count, int) and not isinstance(count, bool): + observations.append({"label": label, "kind": "total_count", "value": count}) + break elif spec["kind"] == "grouped_counts": - expected = spec["values"] for value in documents: if not isinstance(value, Mapping) or not isinstance(value.get("grouped_counts"), Mapping): continue grouped = value["grouped_counts"] - if all( - isinstance(grouped.get(target), Mapping) and grouped[target].get("count") == expected_count - for target, expected_count in expected.items() - ): - matched.append(label) + observed: dict[str, int] = {} + for target in targets: + entry = grouped.get(target) + count = entry.get("count") if isinstance(entry, Mapping) else None + if not isinstance(count, int) or isinstance(count, bool): + break + observed[target] = count + if len(observed) == len(targets): + observations.append({"label": label, "kind": "grouped_counts", "values": observed}) break - if label in matched: - break - return sorted(set(matched)) + return observations + + +def observed_aggregate_labels(observations: Any, aggregates: Any) -> list[str]: + """Compare proxy observations with seed truth inside the post-agent harness.""" + if not isinstance(observations, Sequence) or isinstance(observations, (str, bytes, bytearray)): + return [] + expected_by_label = normalize_evidence_aggregates(aggregates) + matched: set[str] = set() + for observation in observations: + if not isinstance(observation, Mapping): + continue + label = str(observation.get("label") or "") + for expected in expected_by_label.get(label, ()): + if expected["kind"] != observation.get("kind"): + continue + if expected["kind"] == "total_count" and observation.get("value") == expected["value"]: + matched.add(label) + elif expected["kind"] == "grouped_counts" and observation.get("values") == expected["values"]: + matched.add(label) + return sorted(matched) def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, target_ids: Sequence[Any]) -> None: @@ -419,14 +473,17 @@ def set_target_grouped_count_evidence(context: dict[str, Any], values: Mapping[A "decode_evidence_sentinels", "encode_evidence_config", "encode_evidence_sentinels", + "evidence_aggregate_shapes", "fingerprint_evidence_sentinels", "normalize_evidence_fingerprints", + "normalize_evidence_aggregate_shapes", "normalize_evidence_aggregates", "normalize_evidence_sentinels", "normalize_evidence_targets", "observed_sentinel_labels", "observed_fingerprint_labels", "observed_aggregate_labels", + "observed_aggregates", "set_target_count_evidence", "set_target_evidence", "set_target_grouped_count_evidence", diff --git a/evals/proxy.py b/evals/proxy.py index 3a4053c3..2a46e893 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -25,11 +25,11 @@ EVIDENCE_SENTINELS_ENV, consume_evidence_config, fingerprint_evidence_sentinels, - normalize_evidence_aggregates, + normalize_evidence_aggregate_shapes, normalize_evidence_fingerprints, normalize_evidence_sentinels, normalize_evidence_targets, - observed_aggregate_labels, + observed_aggregates, observed_fingerprint_labels, ) from evals.tool_manifest import ToolManifestCapture @@ -45,6 +45,12 @@ REPO_ROOT = Path(__file__).resolve().parent.parent +def proxy_session_log_path(configured_path: Path, *, pid: int | None = None) -> Path: + """Derive the one sidecar owned by this proxy process from the configured base.""" + process_id = os.getpid() if pid is None else pid + return configured_path.with_name(f"{configured_path.name}.{process_id}.jsonl") + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p = argparse.ArgumentParser( description="Stdio MCP recording proxy (tools/call → sidecar JSONL)", @@ -154,7 +160,7 @@ def __init__( if not self.evidence_fingerprints and raw_sentinels: self.evidence_fingerprints = fingerprint_evidence_sentinels(raw_sentinels) self.evidence_targets = normalize_evidence_targets(evidence_targets) - self.evidence_aggregates = normalize_evidence_aggregates(evidence_aggregates) + self.evidence_aggregates = normalize_evidence_aggregate_shapes(evidence_aggregates) self.evidence_active = bool( (self.evidence_fingerprints.keys() | self.evidence_aggregates.keys()) & self.evidence_targets.keys() ) @@ -181,7 +187,8 @@ def __init__( # Post-finalize append attempts (not written; for tests / diagnostics). self.post_finalize_appends = 0 self.log_path.parent.mkdir(parents=True, exist_ok=True) - self.log_path.write_text("", encoding="utf-8") + descriptor = os.open(self.log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) def _append(self, row: dict[str, Any]) -> None: line = json.dumps(row, default=str, ensure_ascii=False) + "\n" @@ -305,24 +312,21 @@ def on_server_message(self, obj: dict[str, Any]) -> None: "seq": pending["seq"], } if self.evidence_active: - # Persist labels only. The matching values and result body stay in memory. - row["observed_sentinels"] = sorted( - set( - observed_fingerprint_labels( - result_text, - self.evidence_fingerprints, - request_args=pending["args"], - evidence_targets=self.evidence_targets, - ) - ) - | set( - observed_aggregate_labels( - result_text, - self.evidence_aggregates, - request_args=pending["args"], - evidence_targets=self.evidence_targets, - ) - ) + # Persist only labels matched from non-enumerable sentinels and + # target-bound aggregate values the agent already received. The + # expected aggregate truth and complete result body never enter + # the proxy process. + row["observed_sentinels"] = observed_fingerprint_labels( + result_text, + self.evidence_fingerprints, + request_args=pending["args"], + evidence_targets=self.evidence_targets, + ) + row["observed_aggregates"] = observed_aggregates( + result_text, + self.evidence_aggregates, + request_args=pending["args"], + evidence_targets=self.evidence_targets, ) if self.record_result_payloads: row["result_text"] = result_text @@ -817,7 +821,7 @@ def request_termination(signum: int, _frame: Any) -> None: try: returncode = run_proxy( list(args.command), - Path(args.log), + proxy_session_log_path(Path(args.log)), record_result_payloads=bool(args.record_result_payloads), evidence_fingerprints=evidence_fingerprints, evidence_targets=evidence_targets, diff --git a/evals/report/identity.py b/evals/report/identity.py index 24d35c22..e9d49489 100644 --- a/evals/report/identity.py +++ b/evals/report/identity.py @@ -121,10 +121,9 @@ def _validate_file(path: Path) -> tuple[FileIdentity, list[str]]: values[field] = next(iter(source)) manifest_values = _record_values(rows, TOOL_MANIFEST_FIELD) - observed_manifests = {value: lines for value, lines in manifest_values.items() if value != MISSING} - if len(observed_manifests) > 1: - issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(observed_manifests)}") - values[TOOL_MANIFEST_FIELD] = next(iter(observed_manifests), MISSING) + if len(manifest_values) > 1: + issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(manifest_values)}") + values[TOOL_MANIFEST_FIELD] = next(iter(manifest_values), MISSING) realized_models = tuple(sorted(_record_values(rows, "model"))) return FileIdentity(path=path, values=values, realized_models=realized_models), issues diff --git a/evals/report/summary.py b/evals/report/summary.py index a0b19d9a..622eb502 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Literal -from evals.results import TaskResult +from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, TaskResult from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result @@ -227,7 +227,9 @@ def summarize( declared_expected_rows = max(declared_expected_rows, row.expected_rows) task_id = row.task_id repetitions_by_task[task_id].add(row.rep) - if not row.trace_integrity: + if row.trace_integrity is False or ( + row.trace_integrity is None and row.schema_version >= TRACE_INTEGRITY_SCHEMA_VERSION + ): trace_invalid_rows += 1 if row.cleanup_error: cleanup_errors += 1 diff --git a/evals/results.py b/evals/results.py index 4212d831..ab433ba0 100644 --- a/evals/results.py +++ b/evals/results.py @@ -14,6 +14,7 @@ from evals.tool_names import split_plane_and_client_calls RESULT_SCHEMA_VERSION = 6 +TRACE_INTEGRITY_SCHEMA_VERSION = 5 TraceIntegrityReason = Literal["recorder_loss", "protocol_violation", "result_pair_mismatch"] @@ -135,7 +136,7 @@ class AgentRun: usage_per_iteration: list[Usage] = field(default_factory=list) cum_input_tokens: int | None = None result_pair_mismatch: bool = False - trace_integrity: bool = True + trace_integrity: bool | None = True trace_integrity_reason: TraceIntegrityReason | None = None tool_manifest_fingerprint: str | None = None token_count_failures: int = 0 @@ -201,7 +202,7 @@ class TaskResult: provider_stop_reason: str | None = None hit_max_iterations: bool = False result_pair_mismatch: bool = False - trace_integrity: bool = True + trace_integrity: bool | None = True trace_integrity_reason: TraceIntegrityReason | None = None tool_manifest_fingerprint: str | None = None token_count_failures: int = 0 @@ -456,7 +457,7 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ), hit_max_iterations=bool(row.get("hit_max_iterations")), result_pair_mismatch=bool(row.get("result_pair_mismatch")), - trace_integrity=bool(row.get("trace_integrity", True)), + trace_integrity=(bool(row["trace_integrity"]) if row.get("trace_integrity") is not None else None), trace_integrity_reason=( str(row["trace_integrity_reason"]) if row.get("trace_integrity_reason") @@ -699,6 +700,7 @@ def agent_run_to_harness_dict( "AGENT_RESULT_COPY_FIELDS", "AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS", "RESULT_SCHEMA_VERSION", + "TRACE_INTEGRITY_SCHEMA_VERSION", "TASK_RESULT_HARNESS_FIELDS", "AgentRun", "CallRecord", diff --git a/evals/runner/live.py b/evals/runner/live.py index 7a38406f..7ceea4b0 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -687,9 +687,14 @@ async def run_live( and 0 <= row.rep < reps ] summary = summarize(result_rows, expected_rows=total_runs, run_keys=run_keys) - if summary.aggregate_n: - rate = summary.aggregate_k / summary.aggregate_n - print(f"success: {summary.aggregate_k}/{summary.aggregate_n} ({rate:.1%})", flush=True) + if summary.task_mean_success is not None: + task_count = sum(task.n > 0 for task in summary.tasks.values()) + print( + f"success: {summary.task_mean_success:.1%} across {task_count} tasks " + f"(cluster-bootstrap95 [{summary.task_cluster_lo:.2f},{summary.task_cluster_hi:.2f}]; " + f"pooled repetitions {summary.aggregate_k}/{summary.aggregate_n})", + flush=True, + ) else: print("success: 0/0 (n/a; no evaluated rows)", flush=True) print(execution_coverage_statement(summary), flush=True) diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 4bad73cf..2165b48c 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -514,7 +514,13 @@ def test_api_driver_records_only_exact_target_bound_aggregate_evidence(): ), Turn( text="", - tool_calls=[ToolCall("e", "count_work_items", {"group_by": "project_id"})], + tool_calls=[ + ToolCall( + "e", + "count_work_items", + {"group_by": "project_id", "project_ids": ["project-1", "project-2"]}, + ) + ], usage=None, stop_reason=StopReason.TOOL_USE, ), @@ -528,7 +534,7 @@ def test_api_driver_records_only_exact_target_bound_aggregate_evidence(): ToolResult(call_id="c", text='{"total_count": 4}'), ToolResult( call_id="d", - text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 4}}}'), + text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 5}}}'), ), ToolResult( call_id="e", diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 1a856d69..98703251 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -4,6 +4,7 @@ import json import os +import shutil import subprocess import sys import textwrap @@ -11,6 +12,7 @@ from pathlib import Path import pytest +import tomllib from evals.drivers import ( AntigravityCliDriver, @@ -708,7 +710,9 @@ def fake_run(cmd, **kwargs): if p.is_file(): bag.setdefault("cfgs", []).append(json.loads(p.read_text())) elif driver_cls is CodexCliDriver: - bag["cmd_joined"] = " ".join(cmd) + codex_home = Path(kwargs["env"]["CODEX_HOME"]) + with (codex_home / "config.toml").open("rb") as stream: + bag["cfg"] = tomllib.load(stream) out = ( json.dumps( { @@ -758,6 +762,92 @@ def fake_run(cmd, **kwargs): assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") +def test_codex_isolated_home_effective_mcp_server_list_is_exactly_plane(tmp_path: Path): + codex_bin = shutil.which("codex") + assert codex_bin is not None, "Codex CLI is required to observe its effective MCP configuration" + + fake_user_home = tmp_path / "fake-user" + global_config = fake_user_home / ".codex" / "config.toml" + global_config.parent.mkdir(parents=True) + global_config.write_text( + '[mcp_servers.forbidden_global]\ncommand = "/usr/bin/false"\n', + encoding="utf-8", + ) + project_config = tmp_path / ".codex" / "config.toml" + project_config.parent.mkdir() + project_config.write_text( + '[mcp_servers.forbidden_project]\ncommand = "/usr/bin/false"\n', + encoding="utf-8", + ) + driver = CodexCliDriver(codex_bin=codex_bin, runner=lambda *_args, **_kwargs: None, allow_live=True) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + effective_env = {**launch.env, "HOME": str(fake_user_home)} + + observed = subprocess.run( + [codex_bin, "mcp", "list", "--json"], + cwd=tmp_path, + env=effective_env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + server_names = sorted(item["name"] for item in json.loads(observed.stdout)) + + assert server_names == ["plane"] + + +def test_opencode_isolated_environment_effective_mcp_server_list_is_exactly_plane(tmp_path: Path): + opencode_bin = shutil.which("opencode") + assert opencode_bin is not None, "OpenCode CLI is required to observe its effective MCP configuration" + + driver = OpencodeCliDriver(opencode_bin=opencode_bin, runner=lambda *_args, **_kwargs: None) + task_state = tmp_path / "task-state" + task_state.mkdir() + launch = driver.write_mcp_config( + task_state, + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + user_config = Path(launch.env["HOME"]) / ".config" / "opencode" / "opencode.json" + user_config.parent.mkdir(parents=True) + user_config.write_text( + json.dumps( + { + "mcp": { + "forbidden_global": { + "type": "local", + "command": ["/usr/bin/false"], + "enabled": True, + } + } + } + ), + encoding="utf-8", + ) + + observed = subprocess.run( + [opencode_bin, "debug", "config"], + cwd=launch.cwd, + env=launch.env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + effective_config = json.loads(observed.stdout) + + assert sorted((effective_config.get("mcp") or {}).keys()) == ["plane"] + + @pytest.mark.parametrize( ("Driver", "bin_key"), [ @@ -767,12 +857,14 @@ def fake_run(cmd, **kwargs): pytest.param(OpencodeCliDriver, "opencode_bin", id="opencode-cwd-config"), ], ) -def test_cli_agent_surfaces_never_contain_evidence_sentinel( +def test_cli_agent_surfaces_never_contain_evidence_truth( tmp_path: Path, Driver: type[CliDriver], bin_key: str, ): sentinel = "hidden-target-fact-7b0a1f9c" + total_count = 918273 + grouped_counts = {"project-1": 564738, "project-2": 102938} seen: dict[str, object] = {} def fake_run(cmd, **kwargs): @@ -783,10 +875,12 @@ def fake_run(cmd, **kwargs): configs.append(json.loads(config_path.read_text())) proxy_args = configs[0]["mcpServers"]["plane"]["args"] elif Driver is CodexCliDriver: - for value in cmd: - prefix = "mcp_servers.plane.args=" - if value.startswith(prefix): - proxy_args = json.loads(value[len(prefix) :]) + codex_home = Path(kwargs["env"]["CODEX_HOME"]) + with (codex_home / "config.toml").open("rb") as stream: + config = tomllib.load(stream) + configs.append(config) + server = config["mcp_servers"]["plane"] + proxy_args = [server["command"], *server["args"]] elif Driver is AntigravityCliDriver: fake_home = Path(kwargs["env"]["HOME"]) for rel in ( @@ -800,6 +894,18 @@ def fake_run(cmd, **kwargs): configs.append(json.loads(config_path.read_text())) proxy_args = configs[0]["mcp"]["plane"]["command"] + if Driver is ClaudeCliDriver: + assert "--strict-mcp-config" in cmd + assert set(configs[0]["mcpServers"]) == {"plane"} + elif Driver is CodexCliDriver: + assert set(configs[0]["mcp_servers"]) == {"plane"} + elif Driver is AntigravityCliDriver: + assert set(configs[0]["mcpServers"]) == {"plane"} + assert all(kwargs["env"].get(name) for name in ("HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME")) + else: + assert set(configs[0]["mcp"]) == {"plane"} + assert all(kwargs["env"].get(name) for name in ("HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME")) + assert proxy_args is not None and "--evidence-file" in proxy_args evidence_path = Path(proxy_args[proxy_args.index("--evidence-file") + 1]) launch_cwd = Path(kwargs["cwd"]).resolve() @@ -840,11 +946,19 @@ def fake_run(cmd, **kwargs): max_turns=1, cwd=tmp_path, evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, - evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1", *grouped_counts]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": total_count}, + {"kind": "grouped_counts", "values": grouped_counts}, + ] + }, ) surface = str(seen["surface"]) assert sentinel not in surface + assert str(total_count) not in surface + assert all(str(count) not in surface for count in grouped_counts.values()) assert EVIDENCE_SENTINELS_ENV not in surface @@ -930,6 +1044,68 @@ def fake_run(cmd, **kwargs): assert run.evidence_trace_available is True +def test_cli_verifier_compares_observed_aggregate_values_after_agent_run(tmp_path): + rows = [ + { + "tool": "count_work_items", + "args": {"project_id": "project-1"}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 1, + "observed_sentinels": [], + "observed_aggregates": [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 3}], + }, + { + "tool": "count_work_items", + "args": {"project_id": "project-1"}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 2, + "observed_sentinels": [], + "observed_aggregates": [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 4}], + }, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 2, + "tool_request_count": 2, + "child_killed": False, + "evidence_trace_available": True, + }, + ] + + def fake_run(cmd, **kwargs): + del kwargs + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + config = json.loads(config_path.read_text(encoding="utf-8")) + proxy_args = config["mcpServers"]["plane"]["args"] + sidecar = Path(proxy_args[proxy_args.index("--log") + 1]) + sidecar.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1"]}, + evidence_aggregates={TARGET_ENTITY_EVIDENCE: [{"kind": "total_count", "value": 4}]}, + ) + + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + + def _timeout_harvest_waits_for_delayed_meta(tmp_path): import threading import time as time_mod diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py index f100d3b7..4720b0a0 100644 --- a/tests/evals/report/test_compare.py +++ b/tests/evals/report/test_compare.py @@ -41,14 +41,14 @@ def test_paired_bootstrap_small_sample_is_task_paired_and_wide(): def test_ab_compare_behaviours(capsys): rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": []}, - {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": []}, + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": [], "trace_integrity": True}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": [], "trace_integrity": True}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": [], "trace_integrity": True}, ] rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": [], "trace_integrity": True}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": [], "trace_integrity": True}, + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": [], "trace_integrity": True}, ] comparison = ab_compare(rows_a, rows_b) @@ -76,14 +76,14 @@ def test_ab_compare_behaviours(capsys): def test_ab_compare_multi_rep_uses_median_successful_call_counts(): rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": []}, - {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": []}, + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": [], "trace_integrity": True}, + {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": [], "trace_integrity": True}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": [], "trace_integrity": True}, ] rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": []}, + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": [], "trace_integrity": True}, + {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": [], "trace_integrity": True}, + {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": [], "trace_integrity": True}, ] comparison = ab_compare(rows_a, rows_b) @@ -103,7 +103,7 @@ def test_ab_compare_excludes_trace_invalid_call_counts(): "calls": [], } ] - rows_b = [{"task_id": "R1", "success": True, "num_calls": 1, "calls": []}] + rows_b = [{"task_id": "R1", "success": True, "trace_integrity": True, "num_calls": 1, "calls": []}] comparison = ab_compare(rows_a, rows_b) diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py index f36abf19..e2629b20 100644 --- a/tests/evals/report/test_identity.py +++ b/tests/evals/report/test_identity.py @@ -276,7 +276,6 @@ def test_report_rejects_manifest_variation_within_one_result_file(tmp_path, caps _row("R1", tool_manifest_fingerprint="manifest-a"), _row("R2", tool_manifest_fingerprint="manifest-b"), ) - _assert_refused( report_mod.main([str(path)]), capsys, @@ -284,6 +283,21 @@ def test_report_rejects_manifest_variation_within_one_result_file(tmp_path, caps ) +def test_report_rejects_mixed_present_and_missing_manifests_within_one_file(tmp_path, capsys): + path = tmp_path / "partially-identified.jsonl" + _write( + path, + _row("R1", tool_manifest_fingerprint="manifest-a"), + _row("R2", tool_manifest_fingerprint=None), + ) + + _assert_refused( + report_mod.main([str(path)]), + capsys, + "", + ) + + def test_report_identifies_but_does_not_refuse_different_tool_manifests(tmp_path, capsys): path_a = tmp_path / "surface-a.jsonl" path_b = tmp_path / "surface-b.jsonl" diff --git a/tests/evals/report/test_load.py b/tests/evals/report/test_load.py index 89724e0a..fa9cc072 100644 --- a/tests/evals/report/test_load.py +++ b/tests/evals/report/test_load.py @@ -86,8 +86,8 @@ def test_load_behaviours(case, tmp_path, capsys): case(tmp_path, capsys) -def test_schema_v0_rows_parse_with_backward_defaults(): - """Synthetic schema-0 rows retain defaults unrelated to this change.""" +def test_schema_v0_rows_parse_with_unknown_trace_integrity(): + """Synthetic schema-0 rows keep data but do not claim verified traces.""" fixture = Path(__file__).parents[2] / "fixtures" / "evals_schema_v0_rows.jsonl" rows = load_rows(fixture) @@ -103,14 +103,16 @@ def test_schema_v0_rows_parse_with_backward_defaults(): assert count_row.final_text.endswith("\n4") assert count_row.result_tokens_estimated is True assert [call.result_tokens for call in count_row.calls] == [315, 64] + assert release_row.trace_integrity is None + assert count_row.trace_integrity is None summary = summarize(rows) assert summary.tasks["L3"].success == "1/1" - assert summary.tasks["L3"].med_calls == 1 + assert summary.tasks["L3"].med_calls is None assert summary.tasks["L3"].result_tokens_mode == "unavailable" assert summary.tasks["R2"].success == "1/1" - assert summary.tasks["R2"].med_calls == 2 - assert summary.tasks["R2"].result_tokens_mode == "estimated" + assert summary.tasks["R2"].med_calls is None + assert summary.tasks["R2"].result_tokens_mode == "unavailable" def test_dedupe_rows_latest_pure(): diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 2e7e0e00..1eef2dd7 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -18,6 +18,7 @@ summarize, wilson_interval, ) +from evals.results import RESULT_SCHEMA_VERSION from tests.evals.conftest import case_params @@ -117,9 +118,34 @@ def test_result_pair_mismatch_preserves_outcome_but_excludes_trace_metrics(): assert summary.tasks["R1"].med_result_tokens is None +def test_missing_trace_integrity_is_unknown_and_current_schema_run_is_incomplete(): + row = { + "schema_version": RESULT_SCHEMA_VERSION, + "task_id": "R1", + "success": True, + "num_calls": 7, + "calls": [{"tool": "unverified", "result_tokens": 99}], + } + + summary = summarize([row], expected_rows=1) + + assert summary.complete is False + assert summary.trace_invalid_rows == 1 + assert summary.tasks["R1"].med_calls is None + assert summary.tasks["R1"].tool_reps == 0 + assert summary.tasks["R1"].med_result_tokens is None + + def _summarize_excludes_infra_errors_from_success(): rows = [ - {"task_id": "R1", "success": True, "num_calls": 2, "calls": [], "error": None}, + { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [], + "error": None, + }, { "task_id": "R1", "success": False, @@ -152,10 +178,10 @@ def _summarize_excludes_infra_errors_from_success(): def _summarize_aggregate_wilson_and_call_variance(): rows = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": []}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": []}, - {"task_id": "R1", "rep": 2, "success": False, "num_calls": 6, "calls": []}, - {"task_id": "R2", "rep": 0, "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "rep": 0, "success": True, "trace_integrity": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "trace_integrity": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "trace_integrity": True, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "trace_integrity": True, "num_calls": 1, "calls": []}, ] s = summarize(rows) assert s.tasks["R1"].n == 3 @@ -181,6 +207,7 @@ def _tool_distribution_uses_successful_repetitions(): "task_id": "R1", "rep": 0, "success": True, + "trace_integrity": True, "num_calls": 3, "calls": [{"tool": "a"}, {"tool": "a"}, {"tool": "b"}], }, @@ -188,6 +215,7 @@ def _tool_distribution_uses_successful_repetitions(): "task_id": "R1", "rep": 1, "success": True, + "trace_integrity": True, "num_calls": 2, "calls": [{"tool": "a"}, {"tool": "c"}], }, @@ -195,6 +223,7 @@ def _tool_distribution_uses_successful_repetitions(): "task_id": "R1", "rep": 2, "success": False, + "trace_integrity": True, "num_calls": 1, "calls": [{"tool": "failed_only"}], }, @@ -202,6 +231,7 @@ def _tool_distribution_uses_successful_repetitions(): "task_id": "R2", "rep": 0, "success": True, + "trace_integrity": True, "num_calls": 1, "calls": [{"tool": "one_rep"}], }, @@ -209,6 +239,7 @@ def _tool_distribution_uses_successful_repetitions(): "task_id": "R3", "rep": 0, "success": False, + "trace_integrity": True, "num_calls": 1, "calls": [{"tool": "failed_only"}], }, @@ -340,7 +371,17 @@ def test_multi_rep_synthetic_file_reports_wilson_and_instability_without_noise_c def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): - rows = [{"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 2, "calls": []}] + rows = [ + { + "task_id": "R1", + "rep": 0, + "label": "local", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [], + } + ] report_mod.print_table(summarize(rows), "Summary: sample.jsonl") diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 29b53562..769572bd 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -35,6 +35,7 @@ def _synth_row( "rep": rep, "label": label, "success": success, + "trace_integrity": True, "num_calls": num_calls, "server": server, "skipped": skipped, @@ -109,6 +110,7 @@ def _report_marks_entirely_estimated_result_token_columns(_tmp_path, capsys): "task_id": "R1", "rep": 0, "success": True, + "trace_integrity": True, "num_calls": 1, "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], "result_tokens_estimated": True, @@ -131,6 +133,7 @@ def _report_marks_mixed_measured_and_estimated_columns(_tmp_path, capsys): "task_id": "R1", "rep": 0, "success": True, + "trace_integrity": True, "num_calls": 1, "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], "result_tokens_estimated": False, @@ -139,6 +142,7 @@ def _report_marks_mixed_measured_and_estimated_columns(_tmp_path, capsys): "task_id": "R1", "rep": 1, "success": True, + "trace_integrity": True, "num_calls": 1, "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], "result_tokens_estimated": True, diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 5c388dbb..9cce1fc8 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -749,6 +749,37 @@ def run_task(self, *args, **kwargs): assert sentinel not in json.dumps(row.to_row()) +def test_run_live_headline_uses_task_cluster_interval(tmp_path, monkeypatch, capsys): + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + del kwargs + return TaskResult(final_text="done", num_calls=1, trace_integrity=True) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + rc = asyncio.run( + run_live( + [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)], + model_alias="standard", + reps=1, + label="local", + out_path=tmp_path / "cluster.jsonl", + ) + ) + + assert rc == 0 + output = capsys.readouterr().out + assert "success: 100.0% across 2 tasks (cluster-bootstrap95 [1.00,1.00]; pooled repetitions 2/2)" in output + assert "success: 2/2 (100.0%)" not in output + + def _run_passes_server_cmd_to_non_claude(tmp_path, monkeypatch, _capsys): from evals.runner import live as run_mod @@ -1203,7 +1234,7 @@ async def verify_ok(*_args, **_kwargs): assert row["cleanup_error"].startswith("TeardownError: 2 cleanup operation(s) failed:") assert delete_calls == [("customer", "customer-1"), ("release_tag", "tag-1")] output = capsys.readouterr().out - assert "success: 1/1 (100.0%)" in output + assert "success: 100.0% across 1 tasks (cluster-bootstrap95 [1.00,1.00]; pooled repetitions 1/1)" in output assert "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)" in output assert "RUN INCOMPLETE:" in output assert "cleanup errors=1" in output diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index c67acbe2..ad44e6c7 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -19,6 +19,8 @@ ensure_proxy_pythonpath, load_proxy_sidecar, load_proxy_sidecar_calls, + proxy_pid_path, + proxy_session_paths, ) from evals.evidence import ( EVIDENCE_SENTINELS_ENV, @@ -44,6 +46,16 @@ REPO = Path(__file__).resolve().parents[2] +def _only_proxy_session(configured_path: Path) -> Path: + sessions = proxy_session_paths(configured_path) + assert len(sessions) == 1, sessions + return sessions[0] + + +def _session_file(configured_path: Path, session_id: str) -> Path: + return configured_path.with_name(f"{configured_path.name}.{session_id}.jsonl") + + def _request(request_id: int, method: str, params: dict | None = None) -> dict: message = {"jsonrpc": "2.0", "id": request_id, "method": method} if params is not None: @@ -197,13 +209,14 @@ def _proxy_records_tools_call_and_exit_code(tmp_path): timeout=15, ) assert proc.returncode == 7 # child exit propagated - assert int(sidecar.with_name(f"{sidecar.name}.pid").read_text(encoding="ascii")) > 0 + session_path = _only_proxy_session(sidecar) + assert int(proxy_pid_path(session_path).read_text(encoding="ascii")) > 0 # Byte-faithful: unparsed line and JSON responses appear on stdout. out = proc.stdout.decode("utf-8", errors="replace") assert "NOT_JSON_LINE" in out assert "list_work_items" in out or "ok:list_work_items" in out - rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + rows = [json.loads(ln) for ln in session_path.read_text(encoding="utf-8").splitlines() if ln.strip()] call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] meta = next(r for r in rows if r.get("row_type") == "proxy_meta") assert len(call_rows) == 2 @@ -324,8 +337,8 @@ def _proxy_exits_when_child_dies_first(tmp_path): # Child's exit code (3) should propagate; tolerate signal map if the # runtime reaps oddly, but meta must still be present. assert rc in (3, 128 + 3) or rc == 3 - assert sidecar.is_file() - text = sidecar.read_text(encoding="utf-8") + assert proxy_session_paths(sidecar) + text = _only_proxy_session(sidecar).read_text(encoding="utf-8") assert "proxy_meta" in text # Prefer exact child code when available if rc not in (3, 128 + 3): @@ -539,7 +552,7 @@ def _proxy_survives_cli_group_kill_and_writes_meta(tmp_path): # Wait until proxy has started (sidecar created) and setsid likely done. boot = time.monotonic() + 5.0 while time.monotonic() < boot: - if sidecar.is_file(): + if proxy_session_paths(sidecar): break time.sleep(0.05) time.sleep(0.5) # allow setsid + optional tools/call @@ -558,16 +571,17 @@ def _proxy_survives_cli_group_kill_and_writes_meta(tmp_path): deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 meta_seen = False while time.monotonic() < deadline: - if sidecar.is_file(): - text = sidecar.read_text(encoding="utf-8") + if proxy_session_paths(sidecar): + text = _only_proxy_session(sidecar).read_text(encoding="utf-8") if "proxy_meta" in text: meta_seen = True break time.sleep(0.1) assert meta_seen, ( - f"proxy_meta missing after group kill; sidecar={sidecar.read_text() if sidecar.is_file() else None!r}" + f"proxy_meta missing after group kill; " + f"sidecar={_only_proxy_session(sidecar).read_text() if proxy_session_paths(sidecar) else None!r}" ) - rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + rows = [json.loads(ln) for ln in _only_proxy_session(sidecar).read_text().splitlines() if ln.strip()] assert rows[-1].get("row_type") == "proxy_meta" finally: if leader.poll() is None: @@ -651,7 +665,11 @@ def test_proxy_signal_finalization_writes_meta_and_preserves_signal_exit(tmp_pat if proc.stdin is not None: proc.stdin.close() - rows = [json.loads(line) for line in sidecar.read_text(encoding="utf-8").splitlines() if line.strip()] + rows = [ + json.loads(line) + for line in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() + if line.strip() + ] assert rows[-1]["row_type"] == "proxy_meta" meta = rows[-1] assert meta["finalization_reason"] == "signal" @@ -749,7 +767,7 @@ def test_proxy_records_exact_target_bound_aggregate_evidence_without_payload(tmp path, evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1"]}, evidence_aggregates={ - TARGET_ENTITY_EVIDENCE: [{"kind": "total_count", "value": 4}], + TARGET_ENTITY_EVIDENCE: [{"kind": "total_count"}], }, ) rec.on_client_message( @@ -773,10 +791,11 @@ def test_proxy_records_exact_target_bound_aggregate_evidence_without_payload(tmp rec.write_meta() calls = load_proxy_sidecar_calls(path) - assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert calls[0]["observed_sentinels"] == [] + assert calls[0]["observed_aggregates"] == [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 4}] persisted = path.read_text(encoding="utf-8") assert "result_text" not in persisted - assert '"total_count"' not in persisted + assert '"content"' not in persisted def _sidecar_result_payload_round_trips_only_when_enabled(tmp_path): @@ -1069,14 +1088,25 @@ def test_scrub_child_pythonpath_removes_repo(): def test_proxy_loads_reusable_private_evidence_file_before_starting_mcp(tmp_path: Path, monkeypatch): sentinel = "hidden-target-fact-7b0a1f9c" + total_count = 918273 + grouped_counts = {"project-1": 564738, "project-2": 102938} evidence_file = tmp_path / "evidence.json" write_evidence_config( evidence_file, {TARGET_ENTITY_EVIDENCE: [sentinel]}, - {TARGET_ENTITY_EVIDENCE: ["target-1"]}, + {TARGET_ENTITY_EVIDENCE: ["target-1", *grouped_counts]}, + { + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": total_count}, + {"kind": "grouped_counts", "values": grouped_counts}, + ] + }, ) assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 - assert sentinel not in evidence_file.read_text(encoding="utf-8") + evidence_payload = evidence_file.read_text(encoding="utf-8") + assert sentinel not in evidence_payload + assert str(total_count) not in evidence_payload + assert all(str(count) not in evidence_payload for count in grouped_counts.values()) captured = {} def fake_run_proxy(command, log_path, **kwargs): @@ -1099,10 +1129,15 @@ def fake_run_proxy(command, log_path, **kwargs): assert rc == 0 assert evidence_file.is_file() - assert sentinel not in evidence_file.read_text(encoding="utf-8") + evidence_payload = evidence_file.read_text(encoding="utf-8") + assert sentinel not in evidence_payload + assert str(total_count) not in evidence_payload + assert all(str(count) not in evidence_payload for count in grouped_counts.values()) assert set(captured["evidence_fingerprints"]) == {TARGET_ENTITY_EVIDENCE} - assert captured["evidence_targets"] == {TARGET_ENTITY_EVIDENCE: ("target-1",)} - assert captured["evidence_aggregates"] == {} + assert captured["evidence_targets"] == {TARGET_ENTITY_EVIDENCE: ("target-1", "project-1", "project-2")} + assert captured["evidence_aggregates"] == { + TARGET_ENTITY_EVIDENCE: ({"kind": "total_count"}, {"kind": "grouped_counts"}) + } def test_probe_then_session_reuses_evidence_config_and_records_provenance(tmp_path: Path): @@ -1115,7 +1150,6 @@ def test_probe_then_session_reuses_evidence_config_and_records_provenance(tmp_pa ) assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 assert sentinel not in evidence_file.read_text(encoding="utf-8") - server = tmp_path / "evidence_server.py" server.write_text( textwrap.dedent( @@ -1186,6 +1220,41 @@ def run_session(sidecar: Path, request: dict) -> tuple[list[dict], dict]: assert sentinel not in evidence_file.read_text(encoding="utf-8") +def test_two_real_proxies_sharing_configured_path_preserve_both_complete_sessions(tmp_path: Path): + server = _write_fake_server(tmp_path / "two_session_server.py") + configured_sidecar = tmp_path / "shared-sidecar.jsonl" + + for request_id, tool in ((1, "first_session_call"), (2, "second_session_call")): + request = _request(request_id, "tools/call", {"name": tool, "arguments": {"session": request_id}}) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(configured_sidecar), + "--", + sys.executable, + str(server), + ], + input=(json.dumps(request) + "\n").encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7, proc.stderr.decode("utf-8", errors="replace") + + calls, status = load_proxy_sidecar(configured_sidecar) + + assert status["state"] == "complete" + assert status["session_file_count"] == 2 + assert status["session_count"] == 2 + assert status["all_sessions_finalized"] is True + assert all(session["finalized"] is True for session in status["sessions"]) + assert all(session["state"] == "complete" for session in status["sessions"]) + assert {call["tool"] for call in calls} == {"first_session_call", "second_session_call"} + + def test_missing_or_malformed_evidence_config_fails_closed(tmp_path: Path): empty = ({}, {}, {}) assert consume_evidence_config(tmp_path / "missing.json") == empty @@ -1263,7 +1332,9 @@ def test_rapid_response_pairing(tmp_path: Path): timeout=30, ) assert proc.returncode == 0, proc.stderr.decode() - rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + rows = [ + json.loads(ln) for ln in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() if ln.strip() + ] call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] meta = next(r for r in rows if r.get("row_type") == "proxy_meta") assert rows[-1].get("row_type") == "proxy_meta" @@ -1335,8 +1406,10 @@ def test_meta_is_last_row_after_forced_kill(tmp_path: Path): timeout=SHUTDOWN_DEADLINE_S + 15, ) elapsed = time_mod.monotonic() - t0 - assert sidecar.is_file() - rows = [json.loads(ln) for ln in sidecar.read_text(encoding="utf-8").splitlines() if ln.strip()] + assert proxy_session_paths(sidecar) + rows = [ + json.loads(ln) for ln in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() if ln.strip() + ] assert rows, "sidecar empty" assert rows[-1].get("row_type") == "proxy_meta" meta = rows[-1] @@ -1399,7 +1472,7 @@ def test_bounded_shutdown_wall_clock(tmp_path: Path): elapsed = time_mod.monotonic() - t0 assert proc.returncode == 0 assert elapsed < SHUTDOWN_DEADLINE_S + 2.0 - rows = [json.loads(ln) for ln in sidecar.read_text().splitlines() if ln.strip()] + rows = [json.loads(ln) for ln in _only_proxy_session(sidecar).read_text().splitlines() if ln.strip()] assert rows[-1].get("row_type") == "proxy_meta" @@ -1546,14 +1619,20 @@ def test_sidecar_rejects_invalid_duplicate_and_gapped_sequences( def test_zero_call_probe_then_real_session_is_complete(tmp_path: Path): path = tmp_path / "probe-then-real.jsonl" manifest = "31c209e40544" + _session_file(path, "probe").write_text( + json.dumps(_complete_proxy_meta(0, manifest)) + "\n", + encoding="utf-8", + ) rows = [ - _complete_proxy_meta(0, manifest), _proxy_call("list_projects", 1), _proxy_call("search_work_items", 2), _proxy_call("create_work_log", 3), _complete_proxy_meta(3, manifest), ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + _session_file(path, "real").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) notes: list[str] = [] result = apply_proxy_sidecar([], [], path, notes) @@ -1572,18 +1651,71 @@ def test_zero_call_probe_then_real_session_is_complete(tmp_path: Path): assert not any("incomplete" in note for note in notes) +def test_zero_proxy_session_files_is_recorder_loss(tmp_path: Path): + path = tmp_path / "never-created.jsonl" + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes, max_wait_s=0) + + assert result.status["state"] == "missing" + assert result.status["session_file_count"] == 0 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + + +def test_single_session_file_rejects_multiple_final_metadata_rows(tmp_path: Path): + path = tmp_path / "duplicate-meta.jsonl" + session = _session_file(path, "one") + session.write_text( + "\n".join(json.dumps(_complete_proxy_meta(0)) for _ in range(2)) + "\n", + encoding="utf-8", + ) + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "incomplete" + assert status["session_file_count"] == 1 + assert status["sessions"][0]["meta_count"] == 2 + assert status["sessions"][0]["finalized"] is False + + +def test_merged_evidence_requires_every_session_to_report_available(tmp_path: Path): + path = tmp_path / "mixed-evidence-availability.jsonl" + unavailable = _complete_proxy_meta(0) + unavailable["evidence_trace_available"] = False + available = _complete_proxy_meta(0) + available["evidence_trace_available"] = True + _session_file(path, "one").write_text(json.dumps(unavailable) + "\n", encoding="utf-8") + _session_file(path, "two").write_text(json.dumps(available) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["all_sessions_finalized"] is True + assert status["evidence_trace_available"] is False + + def test_two_calling_sessions_validate_sequences_independently(tmp_path: Path): path = tmp_path / "two-calling-sessions.jsonl" - rows = [ + first_rows = [ _proxy_call("session-1-call-2", 2), _proxy_call("session-1-call-1", 1), _complete_proxy_meta(2), + ] + second_rows = [ _proxy_call("session-2-call-3", 3), _proxy_call("session-2-call-1", 1), _proxy_call("session-2-call-2", 2), _complete_proxy_meta(3), ] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + _session_file(path, "one").write_text( + "\n".join(json.dumps(row) for row in first_rows) + "\n", + encoding="utf-8", + ) + _session_file(path, "two").write_text( + "\n".join(json.dumps(row) for row in second_rows) + "\n", + encoding="utf-8", + ) calls, status = load_proxy_sidecar(path) @@ -1620,8 +1752,8 @@ def test_trailing_unfinalized_proxy_session_stays_fatal(tmp_path: Path): def test_disagreeing_session_manifests_are_reported_and_invalidated(tmp_path: Path): path = tmp_path / "manifest-disagreement.jsonl" - rows = [_complete_proxy_meta(0, "manifest-a"), _complete_proxy_meta(0, "manifest-b")] - path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + _session_file(path, "one").write_text(json.dumps(_complete_proxy_meta(0, "manifest-a")) + "\n") + _session_file(path, "two").write_text(json.dumps(_complete_proxy_meta(0, "manifest-b")) + "\n") notes: list[str] = [] result = apply_proxy_sidecar([], [], path, notes) diff --git a/uv.lock b/uv.lock index 99ef6309..9a28b821 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anthropic" +version = "0.122.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/23/9987d70b74e3481d5bc5d2021d3e10fd5f60c1f7b54088ea86506d9b7f2b/anthropic-0.122.0.tar.gz", hash = "sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601", size = 1021535, upload-time = "2026-08-13T18:36:00.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/5b/f5c87e71097a9f89f1b414d1ef7ae8439051fae57d5e4ee90946082982b8/anthropic-0.122.0-py3-none-any.whl", hash = "sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67", size = 1041853, upload-time = "2026-08-13T18:36:01.831Z" }, +] + [[package]] name = "anyio" version = "4.12.1" @@ -429,6 +448,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/03/f906829bcfcbb945f19d6a64240ffb66a31d69ca5533e95882f0efc9c13c/cyclopts-4.5.2-py3-none-any.whl", hash = "sha256:ee56ee23c2c81abc34b66b5aa8fd2698ca699740054e84e534449ec3eb7f944d", size = 200165, upload-time = "2026-02-11T16:30:46.942Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -653,6 +681,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d8/b959609e44012a42b1f3e5ba98ea3b33c7e41e6d4b77cd8f00fd19b1d3ad/jiter-0.16.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c", size = 310082, upload-time = "2026-06-29T13:02:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3d/4d7f5667ea0e0548534ba880b84bb3d12924fd133aa83ad6c6c80fca3d76/jiter-0.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b", size = 315643, upload-time = "2026-06-29T13:02:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/9b/83/bed2dcb5c9f3e1ccfcbc67dda48265fe7d5ad0c9cadda5fe95f6e3b87f94/jiter-0.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84", size = 341363, upload-time = "2026-06-29T13:02:34.853Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2f/6bb3c3dda668ebc0445689c81a2b0f26a82b10843d67ed9c9b2c3edc177f/jiter-0.16.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c", size = 365483, upload-time = "2026-06-29T13:02:36.295Z" }, + { url = "https://files.pythonhosted.org/packages/92/35/8a045ccb39164e70dcdae696413b661771f148b68b12b175c3a04d901937/jiter-0.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126", size = 461219, upload-time = "2026-06-29T13:02:38.116Z" }, + { url = "https://files.pythonhosted.org/packages/e7/99/22292dbbf0ed0c610cfe5ddc7f3bd67237a412f121318f865196e62a07bd/jiter-0.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c", size = 374905, upload-time = "2026-06-29T13:02:40.357Z" }, + { url = "https://files.pythonhosted.org/packages/29/ac/2f55ccb1f0eeafa6d89d24caf52f6f0944a59290ee199e9ade62177dca42/jiter-0.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de", size = 348320, upload-time = "2026-06-29T13:02:41.923Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/7d88b9174c40064fabc07c84a9b62e6b10f5644562ec0e0a29392edbe978/jiter-0.16.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244", size = 356519, upload-time = "2026-06-29T13:02:43.436Z" }, + { url = "https://files.pythonhosted.org/packages/27/57/c4a33aeef513a9d5e26e31534e0bcc752d6ea0e54c94ddb7b68bade669c2/jiter-0.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f", size = 394204, upload-time = "2026-06-29T13:02:44.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c6c23e76ebb3766b111bc399437bbc9f870a76e2a92e10b2a5f561d57372/jiter-0.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131", size = 521477, upload-time = "2026-06-29T13:02:46.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/0001c8c0c5976af2625bb1cfb1895e8ec693b6589fe4574b8e6fc2c85501/jiter-0.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b", size = 552187, upload-time = "2026-06-29T13:02:48.144Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/311b718e07e85740e48619c0632b36f7e0b8d113984499e436452ed13a9a/jiter-0.16.0-cp310-cp310-win32.whl", hash = "sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9", size = 206513, upload-time = "2026-06-29T13:02:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/db/7f/ac680eeb0777dc0eb7dc824800ba27880d7f6bc712e362d34ad8ee559f36/jiter-0.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26", size = 199505, upload-time = "2026-06-29T13:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3f/fae6cc967d120ec89e31c5418a51176d8278b3087fbb384a9176754f353c/jiter-0.16.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3", size = 309289, upload-time = "2026-06-29T13:02:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/97c6c3562c077f6247d6e6ce5c82562500b6316c0d928e97e106b7a1321a/jiter-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e", size = 315181, upload-time = "2026-06-29T13:02:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/aa5be86520113b79455c3877f3d1f07a348098df4083ba3688e9537e52dd/jiter-0.16.0-cp311-cp311-win32.whl", hash = "sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7", size = 206755, upload-time = "2026-06-29T13:03:09.653Z" }, + { url = "https://files.pythonhosted.org/packages/64/ec/2feb893eb330bd69b413866f4d5daada33c3962f1c6f270c91ca2d87fdf9/jiter-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1", size = 199155, upload-time = "2026-06-29T13:03:10.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9c/ca040d94415048a3666fc237774df8151c96f8d2b661cbe3b184acc95876/jiter-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056", size = 194403, upload-time = "2026-06-29T13:03:12.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/06/d3/8e278946d43eeca2585b4dd0834a887cd71136329b837f3a16ed86a8b4b0/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad", size = 304518, upload-time = "2026-06-29T13:05:00.172Z" }, + { url = "https://files.pythonhosted.org/packages/72/43/28d4ef495028bf0506a413d4db3f4eb3e7288a382e0f065f306a17bbeb5e/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f", size = 310207, upload-time = "2026-06-29T13:05:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -894,7 +1021,7 @@ wheels = [ [[package]] name = "plane-mcp-server" -version = "0.2.9" +version = "0.2.11" source = { editable = "." } dependencies = [ { name = "authlib" }, @@ -912,33 +1039,37 @@ dev = [ { name = "pytest" }, { name = "ruff" }, ] +evals = [ + { name = "anthropic" }, +] [package.metadata] requires-dist = [ + { name = "anthropic", marker = "extra == 'evals'", specifier = ">=0.121.0" }, { name = "authlib", specifier = ">=1.6.9" }, { name = "boto3", specifier = ">=1.34.0" }, { name = "fakeredis", extras = ["lua"], specifier = ">=2.32.1,<2.35.0" }, { name = "fastmcp", specifier = "==3.2.0" }, { name = "mcp", specifier = "==1.26.0" }, - { name = "plane-sdk", specifier = "==0.2.16" }, + { name = "plane-sdk", specifier = "==0.2.20" }, { name = "py-key-value-aio", extras = ["redis"], specifier = ">=0.4.4,<0.5.0" }, { name = "pyjwt", specifier = ">=2.12.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "evals"] [[package]] name = "plane-sdk" -version = "0.2.16" +version = "0.2.20" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/a5/34db34e8286df62d3080dae04546686d23dc196581b574dff6bd94193baf/plane_sdk-0.2.16.tar.gz", hash = "sha256:0cc881880be107fd0e03db8df17766d72828d4a23d2c1aa3975b45040f67f752", size = 64910, upload-time = "2026-06-15T13:39:09.372Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d8/d9465cdf001aac5988cde173af812b298c85afb8550682fad11ba897112f/plane_sdk-0.2.20.tar.gz", hash = "sha256:d4559e00281be200e386bd322e53dbaafd9f1968bd528bb555f6eda2115518b4", size = 86662, upload-time = "2026-07-20T16:57:41.599Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/f5/049b5ed3286c4c4d4efbd1562bb254b41453888e0a708ca24cd3246289f2/plane_sdk-0.2.16-py3-none-any.whl", hash = "sha256:3f5aaac8b89248c74c2dbfd9902cb4defcfe5a65c430c9239549ce311fcfd999", size = 103856, upload-time = "2026-06-15T13:39:08.385Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8b/c2c52c8c632947e1e12c5612be31bd6180bb6c02046bceb66991f44545ab/plane_sdk-0.2.20-py3-none-any.whl", hash = "sha256:b0a55fec3140025761e8a6c3766b2eed9bea4828ae98c45e3df335cffd1df774", size = 123900, upload-time = "2026-07-20T16:57:40.32Z" }, ] [[package]] @@ -1574,6 +1705,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" From c2326563afb9e68f9083d5260e5131bd7b5f1868 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 09:53:29 +0530 Subject: [PATCH 44/93] Measure off-surface work instead of asserting it cannot happen The evaluated agent holds Plane credentials and shell access, so it could answer a task without going through the surface being measured. Three reviewers named that the largest gap. The architectural fix - a separate server behind a harness-owned proxy, with the agent holding no credential - is designed but is multi-day work, and until it lands the property is an assurance argument rather than a measurement. Every run now reports four named indicators: a success with no Plane call at all, a mutating task that passed without any call that plausibly writes, a correct answer with no target-bound evidence where evidence was configured, and a call count far below the task's own observed distribution. They are named separately because they are different diagnoses, and each names the rows it flagged so a number can be investigated. Mutation intent comes from the catalog's own tags rather than a second list that would drift from the first, and the call-count rule compares against the observed distribution rather than a declared floor - this harness removed author-declared call floors precisely because hand-declared numbers became fiction. A plain outlier fence flagged ordinary one-call variation where the interquartile range is zero, so the rule also requires the count to be at most half the task median. Zero prints explicitly, because a silent absence reads the same as never having checked. The reported limitation is part of the output: this sees off-surface work only when it leaves a trace signature, and cannot see an agent that also makes convincing surface calls. Across 154 recorded live rows it flags nothing. Co-Authored-By: Claude Opus 5 (1M context) --- evals/report/__init__.py | 30 +++ evals/report/compare.py | 4 + evals/report/off_surface.py | 251 +++++++++++++++++++++++++ evals/report/summary.py | 3 + evals/report/table.py | 2 + evals/runner/live.py | 2 + tests/evals/report/test_off_surface.py | 151 +++++++++++++++ tests/evals/report/test_summary.py | 8 + 8 files changed, 451 insertions(+) create mode 100644 evals/report/off_surface.py create mode 100644 tests/evals/report/test_off_surface.py diff --git a/evals/report/__init__.py b/evals/report/__init__.py index 2a22d4df..48df5e33 100644 --- a/evals/report/__init__.py +++ b/evals/report/__init__.py @@ -25,6 +25,22 @@ read_result, validate_run_keys, ) +from .off_surface import ( + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, + INDICATOR_LABELS, + INDICATOR_ORDER, + LOW_CALL_RULE, + OFF_SURFACE_LIMITATION, + WRITE_WITHOUT_WRITE_CALL, + ZERO_CALL_SUCCESS, + OffSurfaceMeasurement, + OffSurfaceRow, + call_plausibly_writes, + measure_off_surface, + off_surface_statement, + task_requires_mutation, +) from .statistics import iqr, median, paired_bootstrap_mean_ci, paired_permutation_pvalue, percentile, wilson_interval from .summary import ( ResultTokensMode, @@ -58,12 +74,23 @@ "ComparabilityError", "FileIdentity", "IdentityReport", + "OffSurfaceMeasurement", + "OffSurfaceRow", "ResultRow", "ResultTokensMode", "Summary", "TaskSummary", + "ANSWER_WITHOUT_PROVENANCE", + "IMPLAUSIBLY_FEW_CALLS", + "INDICATOR_LABELS", + "INDICATOR_ORDER", + "LOW_CALL_RULE", + "OFF_SURFACE_LIMITATION", + "WRITE_WITHOUT_WRITE_CALL", + "ZERO_CALL_SUCCESS", "ab_compare", "build_multi_surface_table", + "call_plausibly_writes", "completeness_statement", "dedupe_rows_latest", "format_multi_rep_surface_cell", @@ -83,6 +110,8 @@ "load_run_expected_rows", "main", "median", + "measure_off_surface", + "off_surface_statement", "paired_bootstrap_mean_ci", "paired_permutation_pvalue", "parse_varied_dimensions", @@ -97,6 +126,7 @@ "result_tokens_mode", "summarize", "surface_label_for_file", + "task_requires_mutation", "task_sort_key", "validate_persisted_identity", "wilson_interval", diff --git a/evals/report/compare.py b/evals/report/compare.py index 49d114dd..24b38fcc 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -7,6 +7,7 @@ from typing import Any from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .off_surface import off_surface_statement from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue from .summary import completeness_statement, execution_coverage_statement, summarize from .table import format_number @@ -137,6 +138,9 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N ) print(f" A {execution_coverage_statement(comparison['summary_a'])}") print(f" B {execution_coverage_statement(comparison['summary_b'])}") + for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): + for line in off_surface_statement(summary.off_surface).splitlines(): + print(f" {label} {line}") print(f" A {completeness_statement(comparison['summary_a'])}") print(f" B {completeness_statement(comparison['summary_b'])}") print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") diff --git a/evals/report/off_surface.py b/evals/report/off_surface.py new file mode 100644 index 00000000..2fd16e42 --- /dev/null +++ b/evals/report/off_surface.py @@ -0,0 +1,251 @@ +"""Trace-signature indicators for possible work outside the measured MCP surface.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, CallRecord, TaskResult +from evals.tasks import TASKS_BY_ID + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import percentile + +ZERO_CALL_SUCCESS = "zero_call_success" +WRITE_WITHOUT_WRITE_CALL = "write_without_write_call" +ANSWER_WITHOUT_PROVENANCE = "answer_without_provenance" +IMPLAUSIBLY_FEW_CALLS = "implausibly_few_calls" + +INDICATOR_ORDER = ( + ZERO_CALL_SUCCESS, + WRITE_WITHOUT_WRITE_CALL, + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, +) +INDICATOR_LABELS = { + ZERO_CALL_SUCCESS: "zero-call success", + WRITE_WITHOUT_WRITE_CALL: "write without a write call", + ANSWER_WITHOUT_PROVENANCE: "answer without provenance", + IMPLAUSIBLY_FEW_CALLS: "implausibly few calls", +} + +LOW_CALL_MIN_REPETITIONS = 5 +LOW_CALL_IQR_MULTIPLIER = 3.0 +LOW_CALL_RULE = ( + "among at least 5 successful trace-usable repetitions for the same task, " + "calls < Q1 - 3×IQR and calls ≤ half the task median" +) +OFF_SURFACE_LIMITATION = ( + "detects off-surface work only when it leaves a trace signature; it cannot detect " + "an agent that performs the work off-surface and also makes convincing surface calls" +) + +_MUTATING_TASK_TAGS = frozenset({"setup", "write"}) +_MUTATING_VERBS = frozenset( + { + "accept", + "add", + "approve", + "archive", + "assign", + "attach", + "cancel", + "complete", + "create", + "decline", + "delete", + "detach", + "disable", + "duplicate", + "enable", + "link", + "manage", + "move", + "publish", + "reject", + "remove", + "restore", + "set", + "start", + "submit", + "transfer", + "unarchive", + "unlink", + "update", + "upload", + } +) + + +@dataclass(frozen=True, slots=True) +class OffSurfaceRow: + """The indicator set attached to one persisted result row.""" + + task_id: str + rep: int + indicators: frozenset[str] + + @property + def address(self) -> str: + return f"{self.task_id}[rep={self.rep}]" + + +@dataclass(frozen=True, slots=True) +class OffSurfaceMeasurement: + """Per-row findings and their run-level aggregate views.""" + + rows: tuple[OffSurfaceRow, ...] = () + + @property + def flagged_rows(self) -> int: + return len(self.rows) + + @property + def indicator_hits(self) -> int: + return sum(len(row.indicators) for row in self.rows) + + def addresses(self, indicator: str) -> tuple[str, ...]: + return tuple(row.address for row in self.rows if indicator in row.indicators) + + +def _trace_usable(row: TaskResult) -> bool: + """Accept authoritative traces and legacy rows predating typed trace integrity.""" + return row.trace_integrity is True or ( + row.trace_integrity is None and row.schema_version < TRACE_INTEGRITY_SCHEMA_VERSION + ) + + +def _successful_row(row: TaskResult) -> bool: + return bool( + row.success and not row.error and not row.skipped and not is_infra_error_row(row) and _trace_usable(row) + ) + + +def task_requires_mutation(task: Mapping[str, Any] | None) -> bool: + """Derive mutation intent from catalog tags instead of a task-id allowlist.""" + tags = task.get("tags") if task is not None else () + return bool(_MUTATING_TASK_TAGS.intersection(str(tag) for tag in (tags or ()))) + + +def call_plausibly_writes(call: CallRecord) -> bool: + """Conservatively recognize successful calls with a mutating verb or action.""" + if call.is_error: + return False + candidates = (call.action, call.tool) + for candidate in candidates: + normalized = str(candidate or "").strip().casefold().replace("-", "_") + verb = normalized.split("_", 1)[0] + if verb in _MUTATING_VERBS: + return True + return False + + +def _has_target_provenance(row: TaskResult) -> bool: + return any(not call.is_error and TARGET_ENTITY_EVIDENCE in (call.observed_sentinels or ()) for call in row.calls) + + +def _answer_was_correct(row: TaskResult) -> bool: + # Provenance-enforced read rows already have success=False when their answer was + # correct but evidence was missing. ``answer_with_provenance`` persists the two + # facts separately in this stable verifier note. + return row.success or "answer_correct=true" in row.verify_note.casefold() + + +def measure_off_surface( + rows: list[ResultRow], + *, + task_catalog: Mapping[str, Mapping[str, Any]] = TASKS_BY_ID, +) -> OffSurfaceMeasurement: + """Compute suspicion indicators without changing row success or completeness.""" + results: list[TaskResult] = [] + for raw_row in rows: + if is_meta_row(raw_row): + continue + results.append(read_result(raw_row)) + + flags_by_index: dict[int, set[str]] = defaultdict(set) + successful_by_task: dict[str, list[tuple[int, TaskResult]]] = defaultdict(list) + for index, row in enumerate(results): + successful = _successful_row(row) + if successful: + successful_by_task[row.task_id].append((index, row)) + if row.num_calls == 0 and not row.calls: + flags_by_index[index].add(ZERO_CALL_SUCCESS) + if task_requires_mutation(task_catalog.get(row.task_id)) and not any( + call_plausibly_writes(call) for call in row.calls + ): + flags_by_index[index].add(WRITE_WITHOUT_WRITE_CALL) + + if ( + _trace_usable(row) + and not row.error + and not row.skipped + and row.evidence_trace_available + and _answer_was_correct(row) + and not _has_target_provenance(row) + ): + flags_by_index[index].add(ANSWER_WITHOUT_PROVENANCE) + + for task_rows in successful_by_task.values(): + if len(task_rows) < LOW_CALL_MIN_REPETITIONS: + continue + call_counts = [float(row.num_calls) for _, row in task_rows] + first_quartile = percentile(call_counts, 0.25) + task_median = percentile(call_counts, 0.5) + third_quartile = percentile(call_counts, 0.75) + assert first_quartile is not None and task_median is not None and third_quartile is not None + lower_outer_fence = first_quartile - LOW_CALL_IQR_MULTIPLIER * (third_quartile - first_quartile) + for index, row in task_rows: + if row.num_calls < lower_outer_fence and row.num_calls <= task_median / 2.0: + flags_by_index[index].add(IMPLAUSIBLY_FEW_CALLS) + + findings = tuple( + OffSurfaceRow( + task_id=row.task_id, + rep=row.rep, + indicators=frozenset(flags_by_index[index]), + ) + for index, row in enumerate(results) + if flags_by_index[index] + ) + return OffSurfaceMeasurement(rows=findings) + + +def off_surface_statement(measurement: OffSurfaceMeasurement) -> str: + """Render an investigation-ready aggregate, including explicit zero results.""" + if measurement.flagged_rows: + headline = ( + f"off-surface indicators: {measurement.flagged_rows} flagged rows " + f"({measurement.indicator_hits} indicator hits)" + ) + else: + headline = "off-surface indicators: 0" + lines = [headline] + for indicator in INDICATOR_ORDER: + addresses = measurement.addresses(indicator) + suffix = f" [{', '.join(addresses)}]" if addresses else "" + rule = f"; rule: {LOW_CALL_RULE}" if indicator == IMPLAUSIBLY_FEW_CALLS else "" + lines.append(f" {INDICATOR_LABELS[indicator]}: {len(addresses)}{suffix}{rule}") + lines.append(f" limitation: {OFF_SURFACE_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "ANSWER_WITHOUT_PROVENANCE", + "IMPLAUSIBLY_FEW_CALLS", + "INDICATOR_LABELS", + "INDICATOR_ORDER", + "LOW_CALL_RULE", + "OFF_SURFACE_LIMITATION", + "OffSurfaceMeasurement", + "OffSurfaceRow", + "WRITE_WITHOUT_WRITE_CALL", + "ZERO_CALL_SUCCESS", + "call_plausibly_writes", + "measure_off_surface", + "off_surface_statement", + "task_requires_mutation", +] diff --git a/evals/report/summary.py b/evals/report/summary.py index 622eb502..23bd12f1 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -10,6 +10,7 @@ from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .off_surface import OffSurfaceMeasurement, measure_off_surface from .statistics import cluster_bootstrap_mean_ci, iqr, median, percentile, wilson_interval ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] @@ -84,6 +85,7 @@ class Summary: unexpected_run_keys: tuple[str, ...] multi_rep: bool result_tokens_mode: ResultTokensMode + off_surface: OffSurfaceMeasurement @property def complete(self) -> bool: @@ -368,4 +370,5 @@ def summarize( unexpected_run_keys=run_keys.unexpected if run_keys is not None else (), multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), + off_surface=measure_off_surface(rows), ) diff --git a/evals/report/table.py b/evals/report/table.py index ddc86bea..feda79f6 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -10,6 +10,7 @@ from evals.tasks import TASKS_BY_ID from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .off_surface import off_surface_statement from .statistics import wilson_interval from .summary import ( Summary, @@ -95,6 +96,7 @@ def print_table(summary: Summary, title: str) -> None: print("task-cluster success: n/a (no evaluated tasks)") print("pooled repetition success: 0/0 (n/a; no evaluated rows)") print(execution_coverage_statement(summary)) + print(off_surface_statement(summary.off_surface)) print(completeness_statement(summary)) if summary.infra_errors: print(f"infra errors: {summary.infra_errors}") diff --git a/evals/runner/live.py b/evals/runner/live.py index 7ceea4b0..9344adaf 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -16,6 +16,7 @@ from evals.drivers.api import MODEL_TIERS from evals.evidence import configured_evidence_labels from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys +from evals.report.off_surface import off_surface_statement from evals.report.summary import completeness_statement, execution_coverage_statement, summarize from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown @@ -698,5 +699,6 @@ async def run_live( else: print("success: 0/0 (n/a; no evaluated rows)", flush=True) print(execution_coverage_statement(summary), flush=True) + print(off_surface_statement(summary.off_surface), flush=True) print(completeness_statement(summary), flush=True) return 0 if summary.complete else 1 diff --git a/tests/evals/report/test_off_surface.py b/tests/evals/report/test_off_surface.py new file mode 100644 index 00000000..1e07c271 --- /dev/null +++ b/tests/evals/report/test_off_surface.py @@ -0,0 +1,151 @@ +"""Offline tests for off-surface trace-signature indicators.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.report import ( + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, + WRITE_WITHOUT_WRITE_CALL, + ZERO_CALL_SUCCESS, + ab_compare, + measure_off_surface, + print_ab_report, + print_table, + summarize, +) +from evals.results import RESULT_SCHEMA_VERSION + + +def _row( + task_id: str, + *, + rep: int = 0, + success: bool = True, + num_calls: int = 1, + calls: list[dict[str, Any]] | None = None, + evidence_trace_available: bool = False, + verify_note: str = "", +) -> dict[str, Any]: + return { + "schema_version": RESULT_SCHEMA_VERSION, + "task_id": task_id, + "rep": rep, + "success": success, + "num_calls": num_calls, + "calls": list(calls or []), + "trace_integrity": True, + "evidence_trace_available": evidence_trace_available, + "verify_note": verify_note, + } + + +def test_zero_call_success_flags_synthetic_bypass_and_clean_row(): + bypass = _row("R1", rep=0, success=True, num_calls=0, calls=[]) + clean = _row("R1", rep=1, success=True, calls=[{"tool": "list_work_items"}]) + trace_invalid = {**_row("R1", rep=2, success=True, num_calls=0, calls=[]), "trace_integrity": False} + + measurement = measure_off_surface([bypass, clean, trace_invalid]) + + assert measurement.addresses(ZERO_CALL_SUCCESS) == ("R1[rep=0]",) + assert all(row.address != "R1[rep=1]" for row in measurement.rows) + assert all(row.address != "R1[rep=2]" for row in measurement.rows) + + # Indicators are measurements, not another pass/fail or completeness policy. + summary = summarize([bypass], expected_rows=1) + assert summary.aggregate_k == summary.aggregate_n == 1 + assert summary.complete is True + assert summary.off_surface.addresses(ZERO_CALL_SUCCESS) == ("R1[rep=0]",) + + +def test_write_without_write_call_uses_catalog_tags(): + suspicious_write = _row("W1", rep=0, calls=[{"tool": "list_labels"}]) + clean_write = _row("W1", rep=1, calls=[{"tool": "manage_work_item_label"}]) + clean_setup = _row("S1", rep=0, calls=[{"tool": "create_work_item_property"}]) + future_setup = _row("FUTURE", rep=0, calls=[{"tool": "list_work_items"}]) + + measurement = measure_off_surface( + [suspicious_write, clean_write, clean_setup, future_setup], + task_catalog={ + "W1": {"tags": {"write"}}, + "S1": {"tags": {"setup"}}, + "FUTURE": {"tags": {"setup"}}, + }, + ) + + assert measurement.addresses(WRITE_WITHOUT_WRITE_CALL) == ("W1[rep=0]", "FUTURE[rep=0]") + + +def test_answer_without_provenance_counts_correct_answer(): + missing = _row( + "R1", + rep=0, + success=False, # Existing provenance enforcement already fails this row. + calls=[{"tool": "retrieve_work_item", "observed_sentinels": []}], + evidence_trace_available=True, + verify_note="answer_correct=true (seed state reported); provenance=missing", + ) + clean = _row( + "R1", + rep=1, + calls=[ + { + "tool": "retrieve_work_item", + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + evidence_trace_available=True, + verify_note="answer_correct=true (seed state reported); provenance=observed", + ) + + measurement = measure_off_surface([missing, clean]) + + assert measurement.addresses(ANSWER_WITHOUT_PROVENANCE) == ("R1[rep=0]",) + + +def test_implausibly_few_calls_uses_observed_outer_fence(): + rows = [ + _row( + "R2", + rep=rep, + num_calls=count, + calls=[{"tool": "list_work_items"}] * count, + ) + for rep, count in enumerate([10, 10, 10, 10, 1]) + ] + + measurement = measure_off_surface(rows) + + assert measurement.addresses(IMPLAUSIBLY_FEW_CALLS) == ("R2[rep=4]",) + assert measure_off_surface(rows[:4]).addresses(IMPLAUSIBLY_FEW_CALLS) == () + benign_dispersion = [ + _row("R2", rep=rep, num_calls=count, calls=[{"tool": "list_work_items"}] * count) + for rep, count in enumerate([3, 3, 3, 3, 2]) + ] + assert measure_off_surface(benign_dispersion).addresses(IMPLAUSIBLY_FEW_CALLS) == () + + +def test_reports_print_explicit_zero_addresses_rule_and_limitation(capsys): + clean = _row("R1", calls=[{"tool": "list_work_items"}]) + print_table(summarize([clean]), "single") + single_output = capsys.readouterr().out + + assert "EXECUTION COVERAGE:" in single_output + assert "off-surface indicators: 0" in single_output + assert "zero-call success: 0" in single_output + assert "calls < Q1 - 3×IQR and calls ≤ half the task median" in single_output + assert "cannot detect an agent" in single_output + assert "RUN COMPLETE:" in single_output + + bypass = _row("W1", rep=3, num_calls=0, calls=[]) + comparison = ab_compare([clean], [bypass]) + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + ab_output = capsys.readouterr().out + + assert "A off-surface indicators: 0" in ab_output + assert "B off-surface indicators: 1 flagged rows" in ab_output + assert "B zero-call success: 1 [W1[rep=3]]" in ab_output + assert "B write without a write call: 1 [W1[rep=3]]" in ab_output diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 1eef2dd7..f1205074 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -390,6 +390,14 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): "task-cluster success: 100.0% across 1 tasks cluster-bootstrap95 [1.00,1.00]\n" "pooled repetition success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "off-surface indicators: 0\n" + " zero-call success: 0\n" + " write without a write call: 0\n" + " answer without provenance: 0\n" + " implausibly few calls: 0; rule: among at least 5 successful trace-usable repetitions for the same " + "task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" + " limitation: detects off-surface work only when it leaves a trace signature; it cannot detect an agent " + "that performs the work off-surface and also makes convincing surface calls\n" "RUN COMPLETE: 1/1 rows completed\n" "tool variability: —\n" "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " From 9feaf7874b28f449327774af5230a240eb642b3f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 12:56:46 +0530 Subject: [PATCH 45/93] Close the reporting gaps, and say only what the tests establish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every report path now prints the off-surface indicators, including --table, which built its footer without them. Zero prints explicitly, because a silent absence is indistinguishable from "not checked". A comparison refuses any surface that lacks a tool_manifest_fingerprint. Missing is a value, not a wildcard — the rule already held within one file and now holds across A/B inputs, so the reporter can no longer print a comparison in which one surface is unidentified. Claude runs under isolated HOME, CLAUDE_CONFIG_DIR and XDG roots rather than inheriting ambient user state, and its transcript is copied to a durable artifact directory before the task directory is destroyed, so the reference a row persists still resolves when someone comes to read it. A failed credentials copy aborts instead of running the battery unauthenticated. The transcript location travels with the launch rather than living on the driver, so a concurrent runner cannot make one task read another's directory. The claims match the evidence. Claude's effective-config exclusivity rests on management readback plus the vendor's documented --strict-mcp-config contract, not on observing the evaluated invocation, and says so. Antigravity 1.1.13 has no introspection command at all, so its exclusivity is labelled unverifiable in the driver and in DESIGN.md rather than implied. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 14 ++ evals/README.md | 14 +- evals/drivers/__init__.py | 2 + evals/drivers/cli/antigravity.py | 8 +- evals/drivers/cli/claude.py | 134 ++++++++++++++++-- evals/drivers/cli/codex.py | 2 + evals/drivers/cli/opencode.py | 3 +- evals/drivers/driver.py | 11 ++ evals/report/identity.py | 9 ++ evals/report/table.py | 9 ++ evals/runner/live.py | 8 ++ tests/evals/drivers/test_cli_driver.py | 180 ++++++++++++++++++++++++- tests/evals/drivers/test_vendors.py | 113 ++++++++++++++-- tests/evals/report/test_identity.py | 30 +++-- tests/evals/report/test_table.py | 29 +++- tests/evals/runner/test_live.py | 1 + 16 files changed, 521 insertions(+), 46 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 7717b2aa..51bbfc74 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -198,6 +198,20 @@ CLI agents already own their model conversation and tool loop, so they implement subprocess, configuration, transcript, and usage differences stay within their driver modules. +CLI MCP configuration is isolated from ambient user state, but the strength of the +effective-config evidence differs by vendor: + +| Driver | Effective-config exclusivity evidence | +|---|---| +| Claude | **Readback-supported, not behaviorally proven for the evaluated invocation.** Real `claude mcp list` reads the same isolated `.claude.json` and observes only `plane`. The evaluated `claude -p` receives that file plus `--strict-mcp-config`; exclusion of project/ambient MCP servers rests on the CLI's documented strict-config contract, not a forbidden-server probe of that invocation. HOME, `CLAUDE_CONFIG_DIR`, and all XDG roots are isolated. | +| Codex | Proven by real `codex mcp list --json` readback under the isolated Codex home. | +| OpenCode | Proven by real `opencode debug config` readback under isolated HOME/XDG roots and the generated project config. | +| Antigravity | **Unverifiable.** Antigravity CLI 1.1.13 has no MCP/effective-config introspection command. The harness isolates HOME/XDG roots and inspects generated files, but neither the harness nor this design treats that as observed effective-config exclusivity. | + +The Antigravity "unverifiable" regression test is documentation coverage: it guards this +claim, not runtime behavior. Separate behavioral tests cover HOME/XDG isolation and generated +file placement, but those still cannot observe Antigravity's effective server set. + Several loop rules are deliberately centralized in `ApiDriver`: - Tool results are paired to model calls by call ID, never by list position. Missing, diff --git a/evals/README.md b/evals/README.md index 302c10ab..3ab84f34 100644 --- a/evals/README.md +++ b/evals/README.md @@ -65,8 +65,8 @@ errors, and observed tool distributions use the same rules as local-server rows. |---|---|---| | `api` | Owned API + MCP loop | Provider-neutral; tiers resolve for `--provider anthropic` (default) or `openai` | | `codex-cli` | OpenAI Codex CLI | `standard` and `fast` resolve to verified GPT-5.6 IDs | -| `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku` | -| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; runs under a synthetic HOME so its MCP config is ours, not yours | +| `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku`; isolated HOME/config/XDG roots and strict MCP config | +| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; isolated HOME/XDG and generated config, but 1.1.13 has no effective-config readback, so exclusivity is unverifiable | | `opencode-cli` | OpenCode | Tiers are intentionally unmapped; pass an explicit ID listed by `opencode models` | ### Model tiers @@ -96,6 +96,11 @@ the row even after its mapping changes. Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool calls are normally counted from the wire rather than from whatever the agent claims it did. If a sidecar is incomplete, the driver can fall back to its CLI event stream or transcript. +Claude transcripts used for fallback are copied out of disposable per-task config into +`.artifacts/claude-cli/` before the row is written, so `driver_raw_ref` remains +resolvable. A file-credential copy failure aborts the task before Claude starts. Refreshed +file credentials are intentionally not copied back into user auth because that would mutate +user state and create cross-task/concurrent refresh races; Claude rows record this limitation. The API driver executes MCP calls itself, records exact result character counts, and sizes result tokens without making a provider request per result. A backend may supply a token @@ -296,8 +301,9 @@ Keep such scripts outside version control — `localdev/` is ignored for exactly compares raw `(task_id, rep)` occurrences with that declaration before latest-wins deduplication, naming missing and unexpected keys (including duplicate excess). - Report headlines use a task-cluster bootstrap interval; the pooled repetition rate and - Wilson interval remain visible but are explicitly labeled as pooled. A/B and surface-table - reports warn when any input lacks a tool-manifest fingerprint. + Wilson interval remain visible but are explicitly labeled as pooled. A/B and multi-file + surface-table reports refuse a comparison when any input lacks a tool-manifest + fingerprint; missing is an explicit unidentified value, not a wildcard. - A feature switched **off for a project** is not a plan gate — it is configuration the harness sets itself, and W11 exists to measure what an agent does when it meets one. - **Gated endpoints returning 402 on a workspace that should work.** Feature flags are diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index c5738996..4f143bd0 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -19,6 +19,7 @@ normalize_claude_usage, parse_claude_json_result, parse_claude_transcript_calls, + prepare_claude_isolated_environment, write_claude_mcp_config, ) from evals.drivers.cli.codex import ( @@ -96,6 +97,7 @@ def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: "parse_codex_jsonl_events", "parse_codex_rollout_calls", "prepare_antigravity_fake_home", + "prepare_claude_isolated_environment", "prepare_codex_home", "prepare_opencode_isolated_environment", "proxy_pid_path", diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index 79528129..fa6a502d 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -94,7 +94,10 @@ class AntigravityCliDriver(CliDriver): Probed 2026-08-12: -p headless, --output-format text|json|stream-json, --model, --dangerously-skip-permissions. MCP only via ~/.gemini/config/mcp_config.json with no CLI flag, hence HOME isolation; no turn-cap flag, so hit_max_turns=False plus a note. - Tool calls come from the proxy sidecar, not from parsing agy stdout. + Tool calls come from the proxy sidecar, not from parsing agy stdout. Antigravity CLI + 1.1.13 has no MCP or effective-config introspection command, so its server exclusivity + cannot be proven by real-binary readback: it is explicitly unverifiable and supported + only by isolated HOME/XDG roots plus inspection of the generated files. """ name = "antigravity-cli" @@ -194,11 +197,12 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], ) -> CliOutput: - del task_cwd, max_turns + del launch, task_cwd, max_turns final_text = (proc.stdout or "").strip() try: if final_text.lstrip().startswith("{"): diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index 7213e6c3..6cf2b231 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -1,16 +1,24 @@ """Claude Code CLI driver and transcript/JSON parsers. -Probed (claude v2.1.228): -p headless; --mcp-config (repeatable) + --strict-mcp-config; +Probed (claude v2.1.232): -p headless; --mcp-config (repeatable) + --strict-mcp-config; --output-format json|text|stream-json; --max-turns (present but hidden from --help); --model; --permission-mode; transcript at -~/.claude/projects//.jsonl, assistant rows carrying -tool_use blocks; MCP tools appear as mcp____. +/projects//.jsonl, assistant rows +carrying tool_use blocks; MCP tools appear as mcp____. The CLI's own help +defines --strict-mcp-config as ignoring every MCP source except --mcp-config. The harness +passes the same isolated .claude.json to that option and to real-binary ``claude mcp list`` +readback, which observes only ``plane``. That readback supports the configuration claim; +exclusion during the evaluated ``claude -p`` invocation rests on the documented strict flag, +not behavioral observation of that invocation. """ from __future__ import annotations import json +import os +import shutil import subprocess +import uuid from collections.abc import Callable from pathlib import Path from typing import Any @@ -87,10 +95,11 @@ def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, return raw, usage_total -def _claude_project_dir(cwd: Path) -> Path: - """Map a cwd to ``~/.claude/projects/`` (``/`` → ``-``).""" +def _claude_project_dir(cwd: Path, *, config_dir: Path | None = None) -> Path: + """Map a cwd to Claude's ``projects/`` transcript directory.""" munged = str(cwd.resolve()).replace("/", "-") - return Path.home() / ".claude" / "projects" / munged + root = config_dir or Path.home() / ".claude" + return root / "projects" / munged def parse_claude_json_result(payload: dict[str, Any] | str) -> dict[str, Any]: @@ -208,15 +217,20 @@ def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]] return calls -def find_claude_transcript(session_id: str | None, cwd: Path) -> Path | None: - """Locate ``~/.claude/projects//.jsonl``.""" +def find_claude_transcript( + session_id: str | None, + cwd: Path, + *, + config_dir: Path | None = None, +) -> Path | None: + """Locate ``/projects//.jsonl``.""" if not session_id: return None - candidate = _claude_project_dir(cwd) / f"{session_id}.jsonl" + candidate = _claude_project_dir(cwd, config_dir=config_dir) / f"{session_id}.jsonl" if candidate.is_file(): return candidate # Fallback: scan project dir for a file containing the session id - proj = _claude_project_dir(cwd) + proj = _claude_project_dir(cwd, config_dir=config_dir) if not proj.is_dir(): return None direct = proj / f"{session_id}.jsonl" @@ -249,15 +263,78 @@ def write_claude_mcp_config( path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") +def persist_claude_transcript( + transcript: Path, + *, + artifact_dir: Path, + session_id: str, +) -> Path: + """Copy a per-task transcript out of disposable Claude state for row forensics.""" + artifact_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + safe_session = "".join(character for character in session_id if character.isalnum() or character in "-_") + destination = artifact_dir / f"{safe_session or 'session'}-{uuid.uuid4().hex}.jsonl" + shutil.copy2(transcript, destination) + destination.chmod(0o600) + return destination + + +def prepare_claude_isolated_environment( + temp_dir: Path, + *, + real_config_dir: Path | None = None, +) -> dict[str, str]: + """Return isolated HOME/config/XDG roots with only Claude's login artifact copied.""" + fake_home = temp_dir / "home" + claude_config = temp_dir / "claude-config" + xdg_roots = { + name: temp_dir / name.lower().replace("_home", "") + for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") + } + for directory in (fake_home, claude_config, *xdg_roots.values()): + directory.mkdir(parents=True, exist_ok=True) + + source_config = real_config_dir or Path(os.environ.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude") + source_credentials = source_config / ".credentials.json" + if source_credentials.is_file(): + try: + shutil.copy2(source_credentials, claude_config / ".credentials.json") + except OSError as exc: + raise RuntimeError( + f"failed to copy Claude credentials into isolated config from {source_credentials}: {exc}" + ) from exc + + return { + **os.environ, + "HOME": str(fake_home), + "CLAUDE_CONFIG_DIR": str(claude_config), + **{name: str(directory) for name, directory in xdg_roots.items()}, + } + + # --------------------------------------------------------------------------- # Claude CLI driver # --------------------------------------------------------------------------- class ClaudeCliDriver(CliDriver): - """Run tasks via ``claude -p`` on the user's Claude Code subscription.""" + """Run Claude Code with isolated state and readback-supported strict MCP config. + + ``--strict-mcp-config`` makes the launch-scoped file exclusive by the Claude CLI's + documented contract, not a behavioral probe of the evaluated ``claude -p`` invocation. + HOME, CLAUDE_CONFIG_DIR, and every XDG root are isolated so ambient user-home state is + not inherited. A real ``claude mcp list`` reads the same temporary .claude.json and + observes exactly the ``plane`` server. + + Known limitation: a refreshed file-based credential is discarded with the per-task + config. It is not copied into user state because doing so would mutate the user's auth + and introduce cross-task/concurrent refresh races; later tasks re-copy the durable source. + """ name = "claude-cli" + run_notes = ( + "known_limitation:claude_file_credentials_refresh_discarded:per-task config is deleted; " + "refresh is not copied to user auth to avoid mutation and cross-task races", + ) temp_dir_prefix = "plane-eval-claude-" def __init__( @@ -293,7 +370,13 @@ def write_mcp_config( server_command: list[str], child_env: dict[str, str], ) -> CliLaunch: - mcp_cfg = temp_dir / "mcp.json" + run_env = prepare_claude_isolated_environment(temp_dir) + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + transcript_config_dir = Path(run_env["CLAUDE_CONFIG_DIR"]) + # Claude's management readback consumes this location, while the session receives + # the exact same physical file through --mcp-config. + mcp_cfg = transcript_config_dir / ".claude.json" write_claude_mcp_config( mcp_cfg, command=server_command[0], @@ -301,7 +384,11 @@ def write_mcp_config( env=child_env, server_name="plane", ) - return CliLaunch(cwd=task_cwd, config_args=["--mcp-config", str(mcp_cfg)]) + return CliLaunch( + cwd=task_cwd, + config_args=["--mcp-config", str(mcp_cfg)], + env=run_env, + ) def build_command( self, @@ -337,6 +424,7 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], @@ -373,8 +461,24 @@ def parse_output( client_calls = list(parsed.get("client_tool_calls") or []) call_source = "json" session_id = parsed.get("session_id") - transcript = find_claude_transcript(session_id, task_cwd) + config_value = (launch.env or {}).get("CLAUDE_CONFIG_DIR") + config_dir = Path(config_value) if config_value else None + transcript = find_claude_transcript( + session_id, + task_cwd, + config_dir=config_dir, + ) if transcript is not None: + if launch.artifact_dir is None: + raise CliOutputError("Claude transcript found without a durable artifact directory") + try: + transcript = persist_claude_transcript( + transcript, + artifact_dir=launch.artifact_dir, + session_id=str(session_id or transcript.stem), + ) + except OSError as exc: + raise CliOutputError(f"failed to persist Claude transcript: {exc}") from exc tagged = parse_claude_transcript_calls(transcript) transcript_plane, transcript_client = split_plane_and_client_calls(tagged) if transcript_plane or transcript_client: @@ -410,5 +514,7 @@ def parse_output( "normalize_claude_usage", "parse_claude_json_result", "parse_claude_transcript_calls", + "persist_claude_transcript", + "prepare_claude_isolated_environment", "write_claude_mcp_config", ] diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index 3b2038de..cf803a2c 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -363,10 +363,12 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], ) -> CliOutput: + del launch del task_cwd, max_turns parsed = parse_codex_jsonl_events(proc.stdout or "") calls = list(parsed.get("calls") or []) diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py index db6abca9..ee39f3dd 100644 --- a/evals/drivers/cli/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -148,11 +148,12 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], ) -> CliOutput: - del task_cwd, max_turns + del launch, task_cwd, max_turns final_text = (proc.stdout or "").strip() # JSONL events: concatenate text-ish fields best-effort. if final_text and "\n" in final_text: diff --git a/evals/drivers/driver.py b/evals/drivers/driver.py index 9e53a55a..a58d755b 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/driver.py @@ -235,7 +235,9 @@ def run_task( evidence_sentinels: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, evidence_aggregates: dict[str, Any] | None = None, + artifact_dir: Path | None = None, ) -> AgentRun: + del artifact_dir if not model: raise ValueError("the API driver requires a model ID") if max_turns < 1: @@ -459,6 +461,7 @@ class CliLaunch: cwd: Path config_args: list[str] = field(default_factory=list) env: dict[str, str] | None = None + artifact_dir: Path | None = None @dataclass @@ -567,6 +570,7 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], @@ -600,6 +604,7 @@ def run_task( evidence_sentinels: dict[str, Any] | None = None, evidence_targets: dict[str, Any] | None = None, evidence_aggregates: dict[str, Any] | None = None, + artifact_dir: Path | None = None, ) -> AgentRun: """Run one CLI task using the shared configuration/proxy/timeout flow.""" task_cwd = (cwd or REPO_ROOT).resolve() @@ -653,6 +658,11 @@ def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: server_command=server_command, child_env=child_env, ) + launch.artifact_dir = ( + artifact_dir + if artifact_dir is not None + else task_cwd / "evals" / "output" / "driver-artifacts" / self.name + ).resolve() command = self.build_command( prompt, model=model, @@ -719,6 +729,7 @@ def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: try: output = self.parse_output( proc, + launch=launch, task_cwd=task_cwd, max_turns=max_turns, notes=notes, diff --git a/evals/report/identity.py b/evals/report/identity.py index e9d49489..83f6ccd4 100644 --- a/evals/report/identity.py +++ b/evals/report/identity.py @@ -152,6 +152,15 @@ def validate_persisted_identity( continue detail = "; ".join(f"{path}={value}" for path, value in by_path.items()) issues.append(f"{field} differs across files: {detail}") + manifest_values = {identity.values[TOOL_MANIFEST_FIELD] for identity in identities} + if MISSING in manifest_values: + unidentified = [ + str(identity.path) for identity in identities if identity.values[TOOL_MANIFEST_FIELD] == MISSING + ] + issues.append( + f"{TOOL_MANIFEST_FIELD} is missing for comparison input(s): {', '.join(unidentified)}; " + "every compared surface must be identified" + ) if issues: raise ComparabilityError(issues) diff --git a/evals/report/table.py b/evals/report/table.py index feda79f6..342b25e2 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -293,6 +293,7 @@ def build_multi_surface_table( "complete": column_summary.complete, "completeness": completeness_statement(column_summary), "coverage": execution_coverage_statement(column_summary), + "off_surface": off_surface_statement(column_summary.off_surface), } return { "columns": columns, @@ -349,6 +350,11 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) lines.append( "| **execution coverage** | | " + " | ".join(footer[column]["coverage"] for column in columns) + " |" ) + lines.append( + "| **off-surface indicators** | | " + + " | ".join(footer[column]["off_surface"].replace("\n", "
") for column in columns) + + " |" + ) lines.append( "| **completeness** | | " + " | ".join(footer[column]["completeness"] for column in columns) + " |" ) @@ -391,6 +397,9 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) ) for column in columns: lines.append(f"{column:12} {footer[column]['coverage']}") + for column in columns: + for line in footer[column]["off_surface"].splitlines(): + lines.append(f"{column:12} {line}") for column in columns: lines.append(f"{column:12} {footer[column]['completeness']}") return "\n".join(lines) + "\n" diff --git a/evals/runner/live.py b/evals/runner/live.py index 9344adaf..e24c62ca 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -101,6 +101,7 @@ async def run_agent_task_via_driver( ctx: dict[str, Any], workspace_slug: str, server_env: dict[str, str] | None = None, + artifact_dir: Path | None = None, ) -> TaskResult: """Run one task through the selected driver.""" project_name = ctx["project_name"] @@ -120,6 +121,7 @@ async def run_agent_task_via_driver( evidence_sentinels=ctx.get("evidence_sentinels"), evidence_targets=ctx.get("evidence_targets"), evidence_aggregates=ctx.get("evidence_aggregates"), + artifact_dir=artifact_dir, ) return agent_run_to_task_result(agent_run) @@ -227,6 +229,7 @@ async def _drive_agent( row: TaskResult, repetition: int, is_api_driver: bool, + artifact_dir: Path, ) -> TaskResult | None: """Run the agent and classify launch or prompt failures.""" # Agent wrap: API failures and CLI failures are infrastructure. @@ -239,6 +242,7 @@ async def _drive_agent( ctx=context, workspace_slug=workspace_slug, server_env=server_env, + artifact_dir=artifact_dir, ) except PromptBindError as exc: # Empty/missing seed IDs in the prompt — not an agent failure. @@ -450,6 +454,7 @@ async def _run_task_repetition( is_api_driver: bool, external: bool, server_env: dict[str, str] | None, + artifact_dir: Path, ) -> TaskResult: """Seed, drive, verify, assemble, and remove one task repetition.""" context: dict[str, Any] = {} @@ -480,6 +485,7 @@ async def _run_task_repetition( row=row, repetition=repetition, is_api_driver=is_api_driver, + artifact_dir=artifact_dir, ) if agent is not None: _apply_agent_run( @@ -554,6 +560,7 @@ async def run_live( battery = battery_fingerprint(tasks) total_runs = len(tasks) * reps out_path.parent.mkdir(parents=True, exist_ok=True) + artifact_dir = out_path.parent / f"{out_path.stem}.artifacts" / driver_name resume_skip: set[tuple[str, int, str]] = set() if resume: @@ -652,6 +659,7 @@ async def run_live( is_api_driver=is_api_driver, external=external, server_env=server_env, + artifact_dir=artifact_dir, ) file.write(json.dumps(row.to_row(), default=str) + "\n") file.flush() diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 98703251..d8bfe804 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -313,11 +313,12 @@ def parse_output( self, proc: subprocess.CompletedProcess[str], *, + launch: CliLaunch, task_cwd: Path, max_turns: int, notes: list[str], ) -> CliOutput: - del proc, task_cwd, max_turns, notes + del proc, launch, task_cwd, max_turns, notes return CliOutput( final_text="done", calls=[ @@ -431,8 +432,8 @@ def build_command(self, prompt, *, model, max_turns, system, launch): del prompt, model, max_turns, system, launch return ["broken-output"] - def parse_output(self, proc, *, task_cwd, max_turns, notes): - del proc, task_cwd, max_turns, notes + def parse_output(self, proc, *, launch, task_cwd, max_turns, notes): + del proc, launch, task_cwd, max_turns, notes raise CliOutputError("cannot parse output") driver: BrokenOutputDriver @@ -803,6 +804,156 @@ def test_codex_isolated_home_effective_mcp_server_list_is_exactly_plane(tmp_path assert server_names == ["plane"] +def test_claude_cli_runs_with_isolated_environment(tmp_path: Path, monkeypatch): + ambient_config = tmp_path / "ambient-claude" + ambient_config.mkdir() + credentials = ambient_config / ".credentials.json" + credentials.write_text('{"sessionKey":"copied-login-only"}\n', encoding="utf-8") + (ambient_config / "settings.json").write_text('{"ambient":true}\n', encoding="utf-8") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient_config)) + + driver = ClaudeCliDriver(runner=lambda *_args, **_kwargs: None) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + + assert launch.env is not None + isolated_roots = [ + Path(launch.env[name]) + for name in ( + "HOME", + "CLAUDE_CONFIG_DIR", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_STATE_HOME", + ) + ] + assert all(root.is_dir() and root.is_relative_to(tmp_path / "task-state") for root in isolated_roots) + isolated_config = Path(launch.env["CLAUDE_CONFIG_DIR"]) + assert (isolated_config / ".credentials.json").read_text(encoding="utf-8") == credentials.read_text( + encoding="utf-8" + ) + assert not (isolated_config / "settings.json").exists() + assert Path(launch.config_args[1]) == isolated_config / ".claude.json" + command = driver.build_command("prompt", model=None, max_turns=1, system=None, launch=launch) + assert "--strict-mcp-config" in command + + +def test_claude_credentials_copy_failure_aborts_before_cli(tmp_path: Path, monkeypatch): + ambient_config = tmp_path / "ambient-claude" + ambient_config.mkdir() + (ambient_config / ".credentials.json").write_text('{"sessionKey":"source"}\n', encoding="utf-8") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient_config)) + invoked = False + + def fake_run(*_args, **_kwargs): + nonlocal invoked + invoked = True + return subprocess.CompletedProcess([], 0, stdout='{"result":"unexpected"}', stderr="") + + def fail_copy(*_args, **_kwargs): + raise OSError("injected credential copy failure") + + monkeypatch.setattr("evals.drivers.cli.claude.shutil.copy2", fail_copy) + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False) + + with pytest.raises(RuntimeError, match="failed to copy Claude credentials into isolated config"): + driver.run_task("prompt", {}, None, 1, cwd=tmp_path) + assert invoked is False + + +def test_claude_credentials_refresh_limitation_reaches_run_notes(tmp_path: Path): + def fake_run(cmd, **_kwargs): + output = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "refresh-limit-session", + "num_turns": 1, + } + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(output), stderr="") + + run = ClaudeCliDriver(runner=fake_run, use_proxy=False).run_task("prompt", {}, None, 1, cwd=tmp_path) + + assert any(note.startswith("known_limitation:claude_file_credentials_refresh_discarded:") for note in run.notes) + + +def test_claude_isolated_environment_management_readback_lists_exactly_plane(tmp_path: Path): + claude_bin = shutil.which("claude") + assert claude_bin is not None, "Claude CLI is required to observe its effective MCP configuration" + + stub = tmp_path / "mcp_stub.py" + stub.write_text( + textwrap.dedent( + """ + import json + import sys + + def send(message): + sys.stdout.write(json.dumps(message) + "\\n") + sys.stdout.flush() + + for line in sys.stdin: + message = json.loads(line) + if "id" not in message: + continue + if message.get("method") == "initialize": + result = { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "serverInfo": {"name": "readback-stub", "version": "1"}, + } + elif message.get("method") == "tools/list": + result = {"tools": []} + else: + result = {} + send({"jsonrpc": "2.0", "id": message["id"], "result": result}) + """ + ), + encoding="utf-8", + ) + driver = ClaudeCliDriver(claude_bin=claude_bin, runner=lambda *_args, **_kwargs: None) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=[sys.executable, str(stub)], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + assert Path(launch.config_args[1]) == Path(launch.env["CLAUDE_CONFIG_DIR"]) / ".claude.json" + + observed = subprocess.run( + [claude_bin, "mcp", "list"], + cwd=tmp_path, + env=launch.env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + server_names = sorted( + line.split(":", 1)[0].strip() + for line in observed.stdout.splitlines() + if ": " in line and not line.startswith("Checking MCP server health") + ) + help_text = subprocess.run( + [claude_bin, "--help"], + text=True, + capture_output=True, + check=True, + timeout=15, + ).stdout + + assert server_names == ["plane"] + assert "Only use MCP servers from --mcp-config" in help_text + assert "ignoring all other MCP configurations" in help_text + + def test_opencode_isolated_environment_effective_mcp_server_list_is_exactly_plane(tmp_path: Path): opencode_bin = shutil.which("opencode") assert opencode_bin is not None, "OpenCode CLI is required to observe its effective MCP configuration" @@ -897,6 +1048,9 @@ def fake_run(cmd, **kwargs): if Driver is ClaudeCliDriver: assert "--strict-mcp-config" in cmd assert set(configs[0]["mcpServers"]) == {"plane"} + assert all( + kwargs["env"].get(name) for name in ("HOME", "CLAUDE_CONFIG_DIR", "XDG_CONFIG_HOME", "XDG_DATA_HOME") + ) elif Driver is CodexCliDriver: assert set(configs[0]["mcp_servers"]) == {"plane"} elif Driver is AntigravityCliDriver: @@ -962,6 +1116,26 @@ def fake_run(cmd, **kwargs): assert EVIDENCE_SENTINELS_ENV not in surface +def test_antigravity_effective_config_exclusivity_is_documented_as_unverifiable(): + driver_doc = AntigravityCliDriver.__doc__ or "" + design = (REPO / "evals" / "DESIGN.md").read_text(encoding="utf-8") + + assert "1.1.13 has no MCP or effective-config introspection command" in driver_doc + assert "explicitly unverifiable" in driver_doc + assert "| Antigravity | **Unverifiable.**" in design + assert "neither the harness nor this design treats that as observed effective-config exclusivity" in design + assert 'The Antigravity "unverifiable" regression test is documentation coverage' in design + + +def test_claude_effective_config_claim_scopes_observation_and_vendor_contract(): + driver_doc = ClaudeCliDriver.__doc__ or "" + design = (REPO / "evals" / "DESIGN.md").read_text(encoding="utf-8") + + assert "not a behavioral probe of the evaluated ``claude -p`` invocation" in driver_doc + assert "Readback-supported, not behaviorally proven for the evaluated invocation" in design + assert "rests on the CLI's documented strict-config contract" in design + + def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index fe3305da..ff84fd28 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -523,7 +523,7 @@ def fake_run(cmd, **kwargs): assert run.usage_total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 -def _claude_driver_falls_back_to_transcript(tmp_path, monkeypatch): +def _claude_driver_falls_back_to_transcript(tmp_path, _monkeypatch): session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" payload = { **CLAUDE_JSON_RESULT, @@ -533,16 +533,16 @@ def _claude_driver_falls_back_to_transcript(tmp_path, monkeypatch): } def fake_run(cmd, **kwargs): + config_dir = Path(kwargs["env"]["CLAUDE_CONFIG_DIR"]) + munged = str(tmp_path.resolve()).replace("/", "-") + project_dir = config_dir / "projects" / munged + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / f"{session_id}.jsonl").write_text( + _transcript_lines(include_tool_search=True), + encoding="utf-8", + ) return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") - # Keep transcript discovery isolated from the developer's real home. - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - munged = str(tmp_path.resolve()).replace("/", "-") - proj = Path.home() / ".claude" / "projects" / munged - proj.mkdir(parents=True, exist_ok=True) - transcript = proj / f"{session_id}.jsonl" - transcript.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") - driver = ClaudeCliDriver(runner=fake_run) run = driver.run_task( "prompt", @@ -555,8 +555,95 @@ def fake_run(cmd, **kwargs): assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] assert run.final_text == "from-json" - # cleanup planted file - transcript.unlink(missing_ok=True) + + +def test_claude_transcript_raw_ref_survives_run_task(tmp_path): + session_id = "cccccccc-dddd-eeee-ffff-000000000001" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], + "result": "from-json", + } + + def fake_run(cmd, **kwargs): + config_dir = Path(kwargs["env"]["CLAUDE_CONFIG_DIR"]) + munged = str(tmp_path.resolve()).replace("/", "-") + project_dir = config_dir / "projects" / munged + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / f"{session_id}.jsonl").write_text( + _transcript_lines(include_tool_search=True), + encoding="utf-8", + ) + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + run = ClaudeCliDriver(runner=fake_run, use_proxy=False).run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, + ) + + assert run.raw_ref is not None + transcript = Path(run.raw_ref) + assert transcript.is_file() + assert [call["tool"] for call in parse_claude_transcript_calls(transcript)] == [ + "ToolSearch", + "list_work_items", + "get_work_item", + ] + + +def test_claude_transcript_lookup_is_launch_local_and_reentrant(tmp_path): + session_id = "same-session-id" + task_cwd = tmp_path / "task-cwd" + task_cwd.mkdir() + driver = ClaudeCliDriver(runner=lambda *_args, **_kwargs: None, use_proxy=False) + + def launch_with_transcript(name: str): + launch = driver.write_mcp_config( + tmp_path / f"state-{name}", + task_cwd=task_cwd, + server_command=["/usr/bin/true"], + child_env={}, + ) + launch.artifact_dir = tmp_path / f"artifacts-{name}" + config_dir = Path((launch.env or {})["CLAUDE_CONFIG_DIR"]) + munged = str(task_cwd.resolve()).replace("/", "-") + transcript_dir = config_dir / "projects" / munged + transcript_dir.mkdir(parents=True) + transcript_dir.joinpath(f"{session_id}.jsonl").write_text( + json.dumps( + { + "message": { + "content": [ + { + "type": "tool_use", + "name": f"mcp__plane__tool_{name}", + "input": {"surface": name}, + } + ] + } + } + ) + + "\n", + encoding="utf-8", + ) + return launch + + launch_a = launch_with_transcript("a") + launch_b = launch_with_transcript("b") + payload = json.dumps({**CLAUDE_JSON_RESULT, "session_id": session_id, "tool_calls": []}) + proc = subprocess.CompletedProcess([], 0, stdout=payload, stderr="") + + output_a = driver.parse_output(proc, launch=launch_a, task_cwd=task_cwd, max_turns=10, notes=[]) + output_b = driver.parse_output(proc, launch=launch_b, task_cwd=task_cwd, max_turns=10, notes=[]) + + assert [call["tool"] for call in output_a.calls] == ["tool_a"] + assert [call["tool"] for call in output_b.calls] == ["tool_b"] + assert output_a.raw_ref is not None and Path(output_a.raw_ref).is_relative_to(launch_a.artifact_dir) + assert output_b.raw_ref is not None and Path(output_b.raw_ref).is_relative_to(launch_b.artifact_dir) def _claude_driver_writes_mcp_config_and_cmd_flags(tmp_path, _monkeypatch): @@ -565,6 +652,7 @@ def _claude_driver_writes_mcp_config_and_cmd_flags(tmp_path, _monkeypatch): def fake_run(cmd, **kwargs): seen["cmd"] = cmd seen["cwd"] = kwargs.get("cwd") + seen["env"] = kwargs.get("env") # Return minimal valid JSON return subprocess.CompletedProcess( cmd, @@ -597,6 +685,9 @@ def fake_run(cmd, **kwargs): assert "--model" in cmd and "sonnet" in cmd assert "--permission-mode" in cmd and "bypassPermissions" in cmd assert "--strict-mcp-config" in cmd + assert seen["env"]["HOME"] != str(Path.home()) + assert seen["env"]["CLAUDE_CONFIG_DIR"] + assert all(seen["env"][name] for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME")) # mcp-config path is a temp file cleaned after run — re-check via write helper cfg = tmp_path / "mcp.json" write_claude_mcp_config( diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py index e2629b20..6f67256b 100644 --- a/tests/evals/report/test_identity.py +++ b/tests/evals/report/test_identity.py @@ -22,6 +22,7 @@ def _row(task_id: str = "R1", **overrides: Any) -> dict[str, Any]: "model": "realized-model", "requested_model": "standard", "requested_tier": "standard", + "tool_manifest_fingerprint": "manifest-a", "success": True, "num_calls": 1, "calls": [], @@ -322,21 +323,30 @@ def test_missing_manifest_observation_is_not_fatal(tmp_path, capsys): assert "TOOL MANIFEST ABSENT" not in output -def test_ab_and_table_warn_prominently_when_any_tool_manifest_is_absent(tmp_path, capsys): +def test_ab_and_table_refuse_when_any_tool_manifest_is_absent(tmp_path, capsys): path_a = tmp_path / "surface-a.jsonl" path_b = tmp_path / "surface-b.jsonl" _write(path_a, _row(tool_manifest_fingerprint="manifest-a")) _write(path_b, _row(tool_manifest_fingerprint=None)) - assert report_mod.main([str(path_a), str(path_b)]) == 0 - ab_output = capsys.readouterr().out - assert "WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified" in ab_output - assert str(path_b) in ab_output - - assert report_mod.main(["--table", str(path_a), str(path_b)]) == 0 - table_output = capsys.readouterr().out - assert "WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified" in table_output - assert str(path_b) in table_output + _assert_refused( + report_mod.main([str(path_a), str(path_b)]), + capsys, + "tool_manifest_fingerprint is missing for comparison input(s)", + ) + + _assert_refused( + report_mod.main(["--table", str(path_a), str(path_b)]), + capsys, + "every compared surface must be identified", + ) + + _write(path_a, _row(tool_manifest_fingerprint=None)) + _assert_refused( + report_mod.main([str(path_a), str(path_b)]), + capsys, + "every compared surface must be identified", + ) def test_exact_run_keys_name_missing_and_duplicate_rows_before_latest_wins(tmp_path, capsys): diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 769572bd..2ae9fe2f 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -42,6 +42,7 @@ def _synth_row( "error": error, "error_class": error_class, "calls": list(calls or []), + "tool_manifest_fingerprint": "manifest-a", } @@ -199,7 +200,8 @@ def _report_main_table_refuses_when_battery_fingerprints_differ(tmp_path, capsys def _report_main_markdown_flag(tmp_path, capsys): f1 = tmp_path / "a.jsonl" - f1.write_text(json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", encoding="utf-8") + row = {**_synth_row("R1", label="candidate", num_calls=1), "tool_manifest_fingerprint": None} + f1.write_text(json.dumps(row) + "\n", encoding="utf-8") rc = report_mod.main(["--table", "--markdown", str(f1)]) assert rc == 0 out = capsys.readouterr().out @@ -340,5 +342,30 @@ def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): "local success task-cluster 100.0% [1.00,1.00]; pooled 1/1 total calls 2 " "tool variability — infra 0\n" "local EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "local off-surface indicators: 0\n" + "local zero-call success: 0\n" + "local write without a write call: 0\n" + "local answer without provenance: 0\n" + "local implausibly few calls: 0; rule: among at least 5 successful trace-usable repetitions for " + "the same task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" + "local limitation: detects off-surface work only when it leaves a trace signature; it cannot detect " + "an agent that performs the work off-surface and also makes convincing surface calls\n" "local RUN COMPLETE: 1/1 rows completed\n" ) + + +def test_multi_surface_table_reports_off_surface_indicators_per_column_in_plain_and_markdown(): + clean = [_synth_row("R1", label="clean", num_calls=1, calls=[{"tool": "list_work_items"}])] + bypass = [_synth_row("W1", label="bypass", num_calls=0, calls=[])] + table = build_multi_surface_table([("clean", clean), ("bypass", bypass)]) + + plain = render_multi_surface_table(table) + assert "clean off-surface indicators: 0" in plain + assert "bypass off-surface indicators: 1 flagged rows (2 indicator hits)" in plain + assert "bypass zero-call success: 1 [W1[rep=0]]" in plain + assert "bypass write without a write call: 1 [W1[rep=0]]" in plain + + markdown = render_multi_surface_table(table, markdown=True) + assert "| **off-surface indicators** | |" in markdown + assert "off-surface indicators: 0
" in markdown + assert "off-surface indicators: 1 flagged rows (2 indicator hits)
" in markdown diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 9cce1fc8..be9e5f23 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -1090,6 +1090,7 @@ class InspectingDriver: def run_task(self, *_args, **kwargs): assert _data_rows(out) == [] assert kwargs["evidence_sentinels"] == {TARGET_ENTITY_EVIDENCE: [seeded_values["sentinel"]]} + assert kwargs["artifact_dir"] == tmp_path / "forensics.artifacts" / "api" return AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") async def verify_ok(_plane, _ctx, _run): From 99166e54f836d6d3c2808f56f26eb6763a8fc7e8 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 13:53:12 +0530 Subject: [PATCH 46/93] Count the calls that failed, not just the calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comparing two tool surfaces is mostly a question about friction, and the comparison could not see it. Call deltas were medians of raw call counts over successful rows, so a surface on which the agent fails schema validation three times and succeeds on the fourth scored identically to one where it succeeded on the first — same pass rate, and the wasted calls read as "more calls" with nothing to say they were rejections. The proxy already recorded is_error per call and the row already persisted errored_calls. Nothing new is measured; what changed is that the comparison reads them. Both the absolute count and the rate are reported, because either alone misleads: a rate improves when extra successful calls dilute a fixed number of failures, and a count carries no sense of how much was attempted. Both are paired by task and bootstrapped over task pairs, matching how the call delta already treats tasks as the sampling unit. Every path that prints the call delta prints these too, zero included, since a silent absence reads as clean when it means unmeasured. is_error is the MCP-level failure flag, so this counts tool-reported failures rather than schema rejections specifically, and the reports say so. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 9 ++ evals/README.md | 6 + evals/report/__init__.py | 14 ++ evals/report/compare.py | 95 ++++++++++-- evals/report/schema_friction.py | 113 ++++++++++++++ evals/report/summary.py | 3 + evals/report/table.py | 11 ++ evals/runner/live.py | 2 + tests/evals/report/test_compare.py | 163 ++++++++++++++++++++- tests/evals/report/test_schema_friction.py | 90 ++++++++++++ tests/evals/report/test_summary.py | 5 + tests/evals/report/test_table.py | 29 ++++ tests/evals/runner/test_live.py | 4 + 13 files changed, 527 insertions(+), 17 deletions(-) create mode 100644 evals/report/schema_friction.py create mode 100644 tests/evals/report/test_schema_friction.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 51bbfc74..6347750d 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -72,6 +72,15 @@ instances under the two labels, independent tasks, and exchangeable A/B labels u permutation null; they do not account for shared environment drift or dependence between tasks. The report prints the paired task count so small samples remain visible. +Errored calls use that same successful, trace-intact row population. Within each task, the +absolute measure is the median errored-call count per repetition (parallel to the call-count +median), while the rate is the task's errored calls divided by its total calls. Cross-task +headlines average task values and paired intervals resample whole task deltas; they do not pool +calls across tasks. Reports retain task IDs for non-zero errors and print measured zeros. A +zero-attempt task has an undefined rate rather than an invented zero rate. `is_error` is the +MCP-level error flag: it counts all tool-reported failures, including an error that is the +correct outcome, and cannot detect an agent that successfully calls the wrong tool. + Single-run success headlines use the same sampling unit: each evaluated task contributes its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. The pooled repetition rate and its Wilson interval remain visible as a descriptive figure, diff --git a/evals/README.md b/evals/README.md index 3ab84f34..d02f2fc8 100644 --- a/evals/README.md +++ b/evals/README.md @@ -158,6 +158,12 @@ deltas. Zero call-delta ties remain in that paired sample. The inference treats independent sampling units and assumes comparable task instances under both labels, so the printed paired task count—and the resulting wide interval for small samples—matters. +Every path that reports call cost also reports errored-call friction on the same successful, +trace-intact rows: an absolute per-task median and an errored/total call rate, with paired task +deltas in A/B output and task IDs for investigation. This is a proxy, not a pure schema-error +counter: MCP `is_error` also marks correct expected failures, while a successful call to the +wrong tool is invisible to it. + Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and fixture names. The battery contains exactly what the agent is asked and no expectation about diff --git a/evals/report/__init__.py b/evals/report/__init__.py index 48df5e33..a341a8a8 100644 --- a/evals/report/__init__.py +++ b/evals/report/__init__.py @@ -41,6 +41,14 @@ off_surface_statement, task_requires_mutation, ) +from .schema_friction import ( + SCHEMA_FRICTION_LIMITATION, + SchemaFrictionMeasurement, + TaskSchemaFriction, + measure_schema_friction, + schema_friction_statement, + successful_trace_rows, +) from .statistics import iqr, median, paired_bootstrap_mean_ci, paired_permutation_pvalue, percentile, wilson_interval from .summary import ( ResultTokensMode, @@ -79,7 +87,9 @@ "ResultRow", "ResultTokensMode", "Summary", + "SchemaFrictionMeasurement", "TaskSummary", + "TaskSchemaFriction", "ANSWER_WITHOUT_PROVENANCE", "IMPLAUSIBLY_FEW_CALLS", "INDICATOR_LABELS", @@ -88,6 +98,7 @@ "OFF_SURFACE_LIMITATION", "WRITE_WITHOUT_WRITE_CALL", "ZERO_CALL_SUCCESS", + "SCHEMA_FRICTION_LIMITATION", "ab_compare", "build_multi_surface_table", "call_plausibly_writes", @@ -111,6 +122,7 @@ "main", "median", "measure_off_surface", + "measure_schema_friction", "off_surface_statement", "paired_bootstrap_mean_ci", "paired_permutation_pvalue", @@ -124,8 +136,10 @@ "render_multi_surface_table", "result_tokens_marker", "result_tokens_mode", + "schema_friction_statement", "summarize", "surface_label_for_file", + "successful_trace_rows", "task_requires_mutation", "task_sort_key", "validate_persisted_identity", diff --git a/evals/report/compare.py b/evals/report/compare.py index 24b38fcc..aae635ba 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -2,12 +2,12 @@ from __future__ import annotations -from collections import defaultdict from pathlib import Path from typing import Any -from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .load import ResultRow, RunKeyValidation from .off_surface import off_surface_statement +from .schema_friction import measure_schema_friction, schema_friction_statement, successful_trace_rows from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue from .summary import completeness_statement, execution_coverage_statement, summarize from .table import format_number @@ -36,15 +36,10 @@ def ab_compare( summary_b = summarize(rows_b, expected_rows=expected_rows_b, run_keys=run_keys_b) def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: - output: dict[str, list[float]] = defaultdict(list) - for raw_row in rows: - row = read_result(raw_row) - if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped or not row.trace_integrity: - continue - if not row.success: - continue - output[row.task_id].append(float(row.num_calls)) - return dict(output) + output: dict[str, list[float]] = {} + for row in successful_trace_rows(rows): + output.setdefault(row.task_id, []).append(float(row.num_calls)) + return output calls_a = successful_calls_by_task(rows_a) calls_b = successful_calls_by_task(rows_b) @@ -78,6 +73,37 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: paired_success_delta = sum(success_deltas) / len(success_deltas) if success_deltas else None paired_success_ci = paired_bootstrap_mean_ci(success_deltas) + friction_a = measure_schema_friction(rows_a) + friction_b = measure_schema_friction(rows_b) + paired_schema_friction: list[dict[str, Any]] = [] + errored_call_deltas: list[float] = [] + errored_call_rate_deltas: list[float] = [] + for task_id in shared: + task_a = friction_a.tasks[task_id] + task_b = friction_b.tasks[task_id] + errored_call_delta = task_b.median_errored_calls - task_a.median_errored_calls + rate_a = task_a.errored_call_rate + rate_b = task_b.errored_call_rate + rate_delta = rate_b - rate_a if rate_a is not None and rate_b is not None else None + errored_call_deltas.append(errored_call_delta) + if rate_delta is not None: + errored_call_rate_deltas.append(rate_delta) + paired_schema_friction.append( + { + "task_id": task_id, + "errored_calls_a": task_a.median_errored_calls, + "errored_calls_b": task_b.median_errored_calls, + "errored_call_delta": errored_call_delta, + "errored_call_rate_a": rate_a, + "errored_call_rate_b": rate_b, + "errored_call_rate_delta": rate_delta, + "raw_errored_calls_a": task_a.errored_calls, + "raw_total_calls_a": task_a.total_calls, + "raw_errored_calls_b": task_b.errored_calls, + "raw_total_calls_b": task_b.total_calls, + } + ) + return { "summary_a": summary_a, "summary_b": summary_b, @@ -91,6 +117,16 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: "n_paired_success": len(success_deltas), "paired_success_delta": paired_success_delta, "paired_success_ci": paired_success_ci, + "paired_schema_friction": paired_schema_friction, + "mean_errored_call_delta": ( + sum(errored_call_deltas) / len(errored_call_deltas) if errored_call_deltas else None + ), + "errored_call_delta_ci": paired_bootstrap_mean_ci(errored_call_deltas), + "mean_errored_call_rate_delta": ( + sum(errored_call_rate_deltas) / len(errored_call_rate_deltas) if errored_call_rate_deltas else None + ), + "errored_call_rate_delta_ci": paired_bootstrap_mean_ci(errored_call_rate_deltas), + "n_paired_errored_call_rates": len(errored_call_rate_deltas), "multi_rep": summary_a.multi_rep or summary_b.multi_rep, "success_a": { "k": summary_a.aggregate_k, @@ -141,6 +177,8 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): for line in off_surface_statement(summary.off_surface).splitlines(): print(f" {label} {line}") + for line in schema_friction_statement(summary.schema_friction).splitlines(): + print(f" {label} {line}") print(f" A {completeness_statement(comparison['summary_a'])}") print(f" B {completeness_statement(comparison['summary_b'])}") print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") @@ -164,6 +202,26 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N f"{probability if probability is not None else 'n/a'} " f"({tie_count} zero-delta ties retained)" ) + errored_delta = comparison["mean_errored_call_delta"] + errored_lo, errored_hi = comparison["errored_call_delta_ci"] + if errored_delta is None or errored_lo is None or errored_hi is None: + print(" mean errored-call delta (B−A): n/a (no paired successful tasks)") + else: + print( + f" mean errored-call delta (B−A): {errored_delta:+.1f} " + f"paired-bootstrap95 [{errored_lo:+.1f},{errored_hi:+.1f}] " + f"(n={comparison['n_paired']} tasks)" + ) + rate_delta = comparison["mean_errored_call_rate_delta"] + rate_lo, rate_hi = comparison["errored_call_rate_delta_ci"] + if rate_delta is None or rate_lo is None or rate_hi is None: + print(" mean errored-call-rate delta (B−A): n/a (no paired tasks with calls on both surfaces)") + else: + print( + f" mean errored-call-rate delta (B−A): {rate_delta * 100:+.1f} percentage points " + f"paired-bootstrap95 [{rate_lo * 100:+.1f},{rate_hi * 100:+.1f}] " + f"(n={comparison['n_paired_errored_call_rates']} tasks)" + ) multiple_repetitions = bool(comparison.get("multi_rep")) if comparison["paired_tasks"]: print() @@ -177,3 +235,18 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N ) else: print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") + if comparison["paired_schema_friction"]: + print() + print("schema friction by paired task (raw errors/calls; median errors per successful repetition):") + for row in comparison["paired_schema_friction"]: + rate_a = row["errored_call_rate_a"] + rate_b = row["errored_call_rate_b"] + rate_a_text = f"{rate_a:.1%}" if rate_a is not None else "n/a" + rate_b_text = f"{rate_b:.1%}" if rate_b is not None else "n/a" + print( + f" {row['task_id']}: " + f"A={row['raw_errored_calls_a']}/{row['raw_total_calls_a']} ({rate_a_text}), " + f"median={row['errored_calls_a']:.1f}; " + f"B={row['raw_errored_calls_b']}/{row['raw_total_calls_b']} ({rate_b_text}), " + f"median={row['errored_calls_b']:.1f}" + ) diff --git a/evals/report/schema_friction.py b/evals/report/schema_friction.py new file mode 100644 index 00000000..edaf8812 --- /dev/null +++ b/evals/report/schema_friction.py @@ -0,0 +1,113 @@ +"""Success-conditioned MCP tool-error measurements for eval reports.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass + +from evals.results import TaskResult + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import median + +SCHEMA_FRICTION_LIMITATION = ( + "limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; " + "an error that is the correct task outcome still contributes, while calling the wrong tool " + "successfully does not" +) + + +@dataclass(frozen=True, slots=True) +class TaskSchemaFriction: + """Absolute and attempt-normalized errors for one task's eligible rows.""" + + task_id: str + repetitions: int + errored_calls: int + total_calls: int + median_errored_calls: float + errored_call_rate: float | None + + @property + def address(self) -> str: + rate = f"{self.errored_call_rate:.1%}" if self.errored_call_rate is not None else "n/a; zero attempts" + return f"{self.task_id}={self.errored_calls}/{self.total_calls} ({rate})" + + +@dataclass(frozen=True, slots=True) +class SchemaFrictionMeasurement: + """Task-cluster aggregate over successful, trace-intact result rows.""" + + tasks: dict[str, TaskSchemaFriction] + task_mean_errored_calls: float | None + task_mean_errored_call_rate: float | None + rate_task_count: int + + @property + def task_count(self) -> int: + return len(self.tasks) + + @property + def errored_task_ids(self) -> tuple[str, ...]: + return tuple(task_id for task_id, task in self.tasks.items() if task.errored_calls) + + +def successful_trace_rows(rows: list[ResultRow]) -> list[TaskResult]: + """Return the exact row population used for successful call deltas.""" + eligible: list[TaskResult] = [] + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped or not row.trace_integrity: + continue + if row.success: + eligible.append(row) + return eligible + + +def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: + """Measure absolute errors and error rate with tasks as sampling units.""" + by_task: dict[str, list[TaskResult]] = defaultdict(list) + for row in successful_trace_rows(rows): + by_task[row.task_id].append(row) + + tasks: dict[str, TaskSchemaFriction] = {} + for task_id in sorted(by_task): + task_rows = by_task[task_id] + errored_calls = sum(row.errored_calls for row in task_rows) + total_calls = sum(row.num_calls for row in task_rows) + tasks[task_id] = TaskSchemaFriction( + task_id=task_id, + repetitions=len(task_rows), + errored_calls=errored_calls, + total_calls=total_calls, + median_errored_calls=float(median([float(row.errored_calls) for row in task_rows]) or 0.0), + errored_call_rate=(errored_calls / total_calls if total_calls else None), + ) + + absolute_values = [task.median_errored_calls for task in tasks.values()] + rate_values = [task.errored_call_rate for task in tasks.values() if task.errored_call_rate is not None] + return SchemaFrictionMeasurement( + tasks=tasks, + task_mean_errored_calls=(sum(absolute_values) / len(absolute_values) if absolute_values else None), + task_mean_errored_call_rate=(sum(rate_values) / len(rate_values) if rate_values else None), + rate_task_count=len(rate_values), + ) + + +def schema_friction_statement(measurement: SchemaFrictionMeasurement) -> str: + """Render explicit zeros, task addresses, and the measurement boundary.""" + absolute = measurement.task_mean_errored_calls + absolute_text = f"{absolute:.1f}" if absolute is not None else "n/a" + rate = measurement.task_mean_errored_call_rate + rate_text = f"{rate:.1%}" if rate is not None else "n/a" + flagged = [task.address for task in measurement.tasks.values() if task.errored_calls] + flagged_text = f" [{', '.join(flagged)}]" if flagged else " []" + return "\n".join( + ( + "schema friction (same successful, trace-intact rows as call deltas): " + f"task-mean median errored calls={absolute_text} across {measurement.task_count} tasks; " + f"task-mean errored-call rate={rate_text} across {measurement.rate_task_count} tasks with calls", + f" errored-call tasks: {len(flagged)}/{measurement.task_count}{flagged_text}", + f" {SCHEMA_FRICTION_LIMITATION}", + ) + ) diff --git a/evals/report/summary.py b/evals/report/summary.py index 23bd12f1..652531cb 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -11,6 +11,7 @@ from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import OffSurfaceMeasurement, measure_off_surface +from .schema_friction import SchemaFrictionMeasurement, measure_schema_friction from .statistics import cluster_bootstrap_mean_ci, iqr, median, percentile, wilson_interval ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] @@ -86,6 +87,7 @@ class Summary: multi_rep: bool result_tokens_mode: ResultTokensMode off_surface: OffSurfaceMeasurement + schema_friction: SchemaFrictionMeasurement @property def complete(self) -> bool: @@ -371,4 +373,5 @@ def summarize( multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), off_surface=measure_off_surface(rows), + schema_friction=measure_schema_friction(rows), ) diff --git a/evals/report/table.py b/evals/report/table.py index 342b25e2..bf37c28d 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -11,6 +11,7 @@ from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import off_surface_statement +from .schema_friction import schema_friction_statement from .statistics import wilson_interval from .summary import ( Summary, @@ -97,6 +98,7 @@ def print_table(summary: Summary, title: str) -> None: print("pooled repetition success: 0/0 (n/a; no evaluated rows)") print(execution_coverage_statement(summary)) print(off_surface_statement(summary.off_surface)) + print(schema_friction_statement(summary.schema_friction)) print(completeness_statement(summary)) if summary.infra_errors: print(f"infra errors: {summary.infra_errors}") @@ -294,6 +296,7 @@ def build_multi_surface_table( "completeness": completeness_statement(column_summary), "coverage": execution_coverage_statement(column_summary), "off_surface": off_surface_statement(column_summary.off_surface), + "schema_friction": schema_friction_statement(column_summary.schema_friction), } return { "columns": columns, @@ -355,6 +358,11 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) + " | ".join(footer[column]["off_surface"].replace("\n", "
") for column in columns) + " |" ) + lines.append( + "| **schema friction** | | " + + " | ".join(footer[column]["schema_friction"].replace("\n", "
") for column in columns) + + " |" + ) lines.append( "| **completeness** | | " + " | ".join(footer[column]["completeness"] for column in columns) + " |" ) @@ -400,6 +408,9 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) for column in columns: for line in footer[column]["off_surface"].splitlines(): lines.append(f"{column:12} {line}") + for column in columns: + for line in footer[column]["schema_friction"].splitlines(): + lines.append(f"{column:12} {line}") for column in columns: lines.append(f"{column:12} {footer[column]['completeness']}") return "\n".join(lines) + "\n" diff --git a/evals/runner/live.py b/evals/runner/live.py index e24c62ca..0c489f3a 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -17,6 +17,7 @@ from evals.evidence import configured_evidence_labels from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys from evals.report.off_surface import off_surface_statement +from evals.report.schema_friction import schema_friction_statement from evals.report.summary import completeness_statement, execution_coverage_statement, summarize from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown @@ -708,5 +709,6 @@ async def run_live( print("success: 0/0 (n/a; no evaluated rows)", flush=True) print(execution_coverage_statement(summary), flush=True) print(off_surface_statement(summary.off_surface), flush=True) + print(schema_friction_statement(summary.schema_friction), flush=True) print(completeness_statement(summary), flush=True) return 0 if summary.complete else 1 diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py index 4720b0a0..e939823e 100644 --- a/tests/evals/report/test_compare.py +++ b/tests/evals/report/test_compare.py @@ -74,22 +74,173 @@ def test_ab_compare_behaviours(capsys): assert "noise floor" not in output +def test_ab_report_distinguishes_identical_success_and_calls_by_errored_calls(capsys): + rows_a = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 4, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + } + ] + rows_b = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 4, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + } + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["success_a"]["k"] == comparison["success_b"]["k"] == 1 + assert comparison["mean_delta"] == 0.0 + assert comparison["mean_errored_call_delta"] == 2.0 + assert comparison["mean_errored_call_rate_delta"] == pytest.approx(0.5) + assert comparison["paired_schema_friction"] == [ + { + "task_id": "R1", + "errored_calls_a": 0.0, + "errored_calls_b": 2.0, + "errored_call_delta": 2.0, + "errored_call_rate_a": 0.0, + "errored_call_rate_b": 0.5, + "errored_call_rate_delta": 0.5, + "raw_errored_calls_a": 0, + "raw_total_calls_a": 4, + "raw_errored_calls_b": 2, + "raw_total_calls_b": 4, + } + ] + + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + output = capsys.readouterr().out + assert "mean errored-call delta (B−A): +2.0" in output + assert "mean errored-call-rate delta (B−A): +50.0 percentage points" in output + assert "R1: A=0/4 (0.0%), median=0.0; B=2/4 (50.0%), median=2.0" in output + assert "is_error is the MCP-level error flag" in output + + +def test_ab_errored_call_rate_delta_averages_paired_tasks_instead_of_pooling_calls(): + rows_a = [ + { + "task_id": "R1", + "success": True, + "num_calls": 1, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R2", + "success": True, + "num_calls": 9, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + ] + rows_b = [ + { + "task_id": "R1", + "success": True, + "num_calls": 1, + "errored_calls": 1, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R2", + "success": True, + "num_calls": 9, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["mean_errored_call_delta"] == 0.5 + assert comparison["mean_errored_call_rate_delta"] == 0.5 + assert comparison["mean_errored_call_rate_delta"] != pytest.approx(1 / 10) + assert comparison["n_paired_errored_call_rates"] == 2 + assert comparison["errored_call_rate_delta_ci"] == pytest.approx((0.0375, 0.9625)) + + def test_ab_compare_multi_rep_uses_median_successful_call_counts(): rows_a = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 1, "calls": [], "trace_integrity": True}, - {"task_id": "R1", "rep": 1, "success": False, "num_calls": 9, "calls": [], "trace_integrity": True}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 5, "calls": [], "trace_integrity": True}, + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 1, + "success": False, + "num_calls": 9, + "errored_calls": 99, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 2, + "success": True, + "num_calls": 5, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + }, ] rows_b = [ - {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": [], "trace_integrity": True}, - {"task_id": "R1", "rep": 1, "success": True, "num_calls": 4, "calls": [], "trace_integrity": True}, - {"task_id": "R1", "rep": 2, "success": True, "num_calls": 6, "calls": [], "trace_integrity": True}, + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 2, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 4, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 2, + "success": True, + "num_calls": 6, + "errored_calls": 4, + "calls": [], + "trace_integrity": True, + }, ] comparison = ab_compare(rows_a, rows_b) assert comparison["multi_rep"] is True assert comparison["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] + assert comparison["mean_errored_call_delta"] == 1.0 + assert comparison["mean_errored_call_rate_delta"] == pytest.approx(1 / 6) def test_ab_compare_excludes_trace_invalid_call_counts(): diff --git a/tests/evals/report/test_schema_friction.py b/tests/evals/report/test_schema_friction.py new file mode 100644 index 00000000..37500607 --- /dev/null +++ b/tests/evals/report/test_schema_friction.py @@ -0,0 +1,90 @@ +"""Offline tests for success-conditioned MCP error measurements.""" + +from __future__ import annotations + +import pytest + +from evals.report import measure_schema_friction, schema_friction_statement + + +def _row(task_id: str, **overrides): + row = { + "task_id": task_id, + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 4, + "errored_calls": 1, + "calls": [], + } + row.update(overrides) + return row + + +def test_schema_friction_uses_exact_successful_call_delta_population(): + measurement = measure_schema_friction( + [ + _row("R1", errored_calls=2), + _row("R2", success=False, errored_calls=20), + _row("R3", trace_integrity=False, errored_calls=30), + _row("R4", error="harness failed", errored_calls=40), + _row("R5", error_class="infra_cli", errored_calls=50), + _row("R6", skipped="env:plan-gated:feature", errored_calls=60), + { + "row_type": "meta", + "expected_rows": 6, + "success": True, + "trace_integrity": True, + "num_calls": 4, + "errored_calls": 70, + }, + ] + ) + + assert list(measurement.tasks) == ["R1"] + assert measurement.tasks["R1"].errored_calls == 2 + assert measurement.tasks["R1"].total_calls == 4 + assert measurement.task_mean_errored_calls == 2.0 + assert measurement.task_mean_errored_call_rate == 0.5 + + +def test_schema_friction_rate_is_task_mean_instead_of_pooled_call_rate(): + measurement = measure_schema_friction( + [ + _row("R1", num_calls=1, errored_calls=1), + _row("R2", num_calls=9, errored_calls=0), + ] + ) + + assert measurement.task_mean_errored_calls == 0.5 + assert measurement.task_mean_errored_call_rate == 0.5 + assert measurement.task_mean_errored_call_rate != pytest.approx(1 / 10) + + +def test_schema_friction_absolute_count_is_per_task_median_across_repetitions(): + measurement = measure_schema_friction( + [ + _row("R1", rep=0, num_calls=10, errored_calls=0), + _row("R1", rep=1, num_calls=10, errored_calls=0), + _row("R1", rep=2, num_calls=10, errored_calls=9), + ] + ) + + assert measurement.tasks["R1"].median_errored_calls == 0.0 + assert measurement.tasks["R1"].errored_calls == 9 + assert measurement.tasks["R1"].errored_call_rate == pytest.approx(0.3) + + +def test_schema_friction_prints_zero_and_does_not_invent_zero_attempt_rate(): + statement = schema_friction_statement(measure_schema_friction([_row("R1", num_calls=0, errored_calls=0)])) + + assert "task-mean median errored calls=0.0 across 1 tasks" in statement + assert "task-mean errored-call rate=n/a across 0 tasks with calls" in statement + assert "errored-call tasks: 0/1 []" in statement + assert "is_error is the MCP-level error flag" in statement + assert "correct task outcome still contributes" in statement + assert "wrong tool successfully does not" in statement + + no_data = schema_friction_statement(measure_schema_friction([])) + assert "task-mean median errored calls=n/a across 0 tasks" in no_data + assert "errored-call tasks: 0/0 []" in no_data diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index f1205074..ddd1f4e8 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -398,6 +398,11 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): "task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" " limitation: detects off-surface work only when it leaves a trace signature; it cannot detect an agent " "that performs the work off-surface and also makes convincing surface calls\n" + "schema friction (same successful, trace-intact rows as call deltas): task-mean median errored calls=0.0 " + "across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" + " errored-call tasks: 0/1 []\n" + " limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an error that is " + "the correct task outcome still contributes, while calling the wrong tool successfully does not\n" "RUN COMPLETE: 1/1 rows completed\n" "tool variability: —\n" "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 2ae9fe2f..6213d98a 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -350,6 +350,11 @@ def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): "the same task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" "local limitation: detects off-surface work only when it leaves a trace signature; it cannot detect " "an agent that performs the work off-surface and also makes convincing surface calls\n" + "local schema friction (same successful, trace-intact rows as call deltas): task-mean median errored " + "calls=0.0 across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" + "local errored-call tasks: 0/1 []\n" + "local limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an " + "error that is the correct task outcome still contributes, while calling the wrong tool successfully does not\n" "local RUN COMPLETE: 1/1 rows completed\n" ) @@ -369,3 +374,27 @@ def test_multi_surface_table_reports_off_surface_indicators_per_column_in_plain_ assert "| **off-surface indicators** | |" in markdown assert "off-surface indicators: 0
" in markdown assert "off-surface indicators: 1 flagged rows (2 indicator hits)
" in markdown + + +def test_multi_surface_table_reports_schema_friction_per_column_in_plain_and_markdown(): + clean = [_synth_row("R1", label="clean", num_calls=4, calls=[{"tool": "get_work_item"}])] + friction = [ + _synth_row( + "R1", + label="friction", + num_calls=4, + calls=[{"tool": "get_work_item", "is_error": True}], + ) + ] + table = build_multi_surface_table([("clean", clean), ("friction", friction)]) + + plain = render_multi_surface_table(table) + assert "clean schema friction" in plain + assert "clean errored-call tasks: 0/1 []" in plain + assert "friction errored-call tasks: 1/1 [R1=1/4 (25.0%)]" in plain + assert "friction limitation: is_error is the MCP-level error flag" in plain + + markdown = render_multi_surface_table(table, markdown=True) + assert "| **schema friction** | |" in markdown + assert "errored-call tasks: 0/1 []" in markdown + assert "errored-call tasks: 1/1 [R1=1/4 (25.0%)]" in markdown diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index be9e5f23..9a62204a 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -861,6 +861,10 @@ async def fake_drive(**kwargs): assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed assert "finished 2/2 in " in printed assert "2 pass, 0 fail, 0 skip" in printed + assert "schema friction (same successful, trace-intact rows as call deltas)" in printed + assert "task-mean median errored calls=0.0 across 2 tasks" in printed + assert "errored-call tasks: 0/2 []" in printed + assert "is_error is the MCP-level error flag" in printed _RUN_CASES = case_params( From dbcc6a54a3fafa2314f10ee0a30c8693fb664e88 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 14:08:06 +0530 Subject: [PATCH 47/93] Say what the fixture seed actually guarantees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The design claimed a repetition's fixture truth is reproducible from its persisted seed. It is not: seeding reads the current date for cycle and work-item dates, and an identifier collision retries with fresh randomness drawn outside the seeded namespace, so the same seed on another day builds a different fixture. What the seed does guarantee is the property it was built for and the one the harness depends on — each repetition's sentinels are independent, so no repetition can leak another's answer. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 6347750d..21ff2957 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -254,8 +254,15 @@ are not inherited. Result rows retain only seeded entity kinds and randomization namespaces. They never contain target entity IDs or randomized truth values. Each repetition has an independent persisted -`fixture_seed_id`; its truth is reproducible from that seed plus namespace without exposing -any later repetition's independent sentinel. +`fixture_seed_id`, and the randomized truth it derives is recoverable from that seed plus +namespace without exposing any later repetition's independent sentinel. + +That seed is not a replay recipe for the whole fixture. Seeding also reads `date.today()` for +cycle and work-item dates, and identifier collisions retry with fresh `secrets` randomness +drawn outside the seeded namespace, so re-running a seed on another day — or after a +collision — does not reconstruct the same fixture. What the seed guarantees is narrower and is +what it was built for: each repetition's sentinels are independent, so no repetition can leak +another's answer. The first line of a new result file is a meta row containing the run identity, label, server, battery, requested model/tier, resolved model, driver, provider, Git SHA, exact task-id list, From 054cf729679f82c0437b892b7e6891a2e593c0b1 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 19:45:43 +0530 Subject: [PATCH 48/93] Let the isolated Codex home actually make tool calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `codex exec` is non-interactive, so an MCP call that raises an approval request has nobody to answer it and Codex cancels its own call with "user cancelled MCP tool call". The agent then answers from nothing. That is not hypothetical: a live run recorded zero Plane calls on every task across three drivers while still emitting confident answers — one row reported "count: 3" for an urgent-work-item count it never queried. Verifiers caught it through missing target-bound provenance, so nothing scored as a false pass, but the battery was measuring the model's imagination rather than the tool surface. Config isolation is what severed it. The developer config routes approvals through automatic review; an isolated home inherits nothing, which is the point of isolation, so it has to say so itself. The test asserts the setting is present, because 562 passing tests said nothing about a run that made no calls at all. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/cli/codex.py | 11 ++++++++++- tests/evals/drivers/test_cli_driver.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index cf803a2c..51e4b688 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -232,8 +232,17 @@ def write_codex_mcp_config( env: dict[str, str], server_name: str = "plane", ) -> None: - """Write the complete MCP config for an isolated Codex home.""" + """Write the complete MCP config for an isolated Codex home. + + ``approvals_reviewer`` is required, not cosmetic. ``codex exec`` is non-interactive, so an + MCP call that raises an approval request has nobody to answer it and Codex cancels its own + call with ``user cancelled MCP tool call``. The agent then answers from nothing: a live run + recorded zero calls on every task while still emitting confident answers. Routing approvals + through automatic review is what the developer config already does; an isolated home has to + say so itself, because isolation is exactly what stops it being inherited. + """ lines = [ + 'approvals_reviewer = "auto_review"', f"[mcp_servers.{json.dumps(server_name)}]", f"command = {json.dumps(command)}", f"args = {json.dumps(args)}", diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index d8bfe804..3ed5fa81 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -24,6 +24,7 @@ harvest_proxy_after_cli_timeout, load_proxy_sidecar, load_proxy_sidecar_calls, + prepare_codex_home, proxy_pid_path, proxy_wrap_server_command, run_cli_subprocess, @@ -1559,3 +1560,23 @@ def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): assert len(calls) == 1 assert src == "proxy" assert any("incomplete" in n for n in notes) + + +def test_codex_isolated_home_routes_approvals_through_automatic_review(tmp_path: Path): + """An isolated home must permit unattended MCP calls, or the battery measures nothing. + + ``codex exec`` is non-interactive: an MCP call that raises an approval request is cancelled + by Codex itself with ``user cancelled MCP tool call``, and the agent then answers without + touching the surface. A live run recorded zero calls on all four tasks while still emitting + confident answers, and 562 passing tests said nothing about it. + """ + codex_home = tmp_path / "codex-home" + prepare_codex_home( + codex_home, + command="/usr/bin/true", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "k"}, + real_codex_home=tmp_path / "absent", + ) + config = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approvals_reviewer = "auto_review"' in config From 974c097b4e7c0b3963c3249751b47cc96a9b24dd Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Sun, 16 Aug 2026 21:50:01 +0530 Subject: [PATCH 49/93] Stop demanding a tool manifest from rows that never reached the surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A repetition that fails during seeding never launches an agent, so no tools/list is ever observed and there is no manifest to record. Requiring one anyway made mixed present/missing look like a mismatch and refused the comparison outright. That is not theoretical: the first live A/B was refused because six rows — L2 and L5, identical on both surfaces, failing in seeding — carried no fingerprint. Every statistic in the report already excludes those rows, so the refusal blocked a comparison that was sound in every respect it was meant to protect. Identity is now read from the rows that actually reached the surface. The rule the refusal exists for is unchanged: among rows that ran, a missing manifest is still a value rather than a wildcard, and a surface without one still cannot be compared. Co-Authored-By: Claude Opus 5 (1M context) --- evals/report/identity.py | 9 ++++++- tests/evals/report/test_identity.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/evals/report/identity.py b/evals/report/identity.py index 83f6ccd4..7813be85 100644 --- a/evals/report/identity.py +++ b/evals/report/identity.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import Any +from evals.report.load import is_infra_error_row + MISSING = "" IDENTITY_FIELDS = ("battery", "resolved_model", "provider", "driver", "server") VARYABLE_DIMENSIONS = ("resolved_model", "provider", "driver", "server") @@ -120,7 +122,12 @@ def _validate_file(path: Path) -> tuple[FileIdentity, list[str]]: source = row_values or header_values or {MISSING: []} values[field] = next(iter(source)) - manifest_values = _record_values(rows, TOOL_MANIFEST_FIELD) + # A row that failed in seeding never launched an agent, so no tools/list was ever + # observed and there is no manifest to record. Demanding one from those rows refuses + # comparisons that are perfectly sound: the first live A/B was blocked by six + # infra_seed rows that every statistic already excludes. + surface_rows = [(line, record) for line, record in rows if not is_infra_error_row(record)] + manifest_values = _record_values(surface_rows, TOOL_MANIFEST_FIELD) if len(manifest_values) > 1: issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(manifest_values)}") values[TOOL_MANIFEST_FIELD] = next(iter(manifest_values), MISSING) diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py index 6f67256b..c914d68e 100644 --- a/tests/evals/report/test_identity.py +++ b/tests/evals/report/test_identity.py @@ -7,6 +7,7 @@ from typing import Any from evals import report as report_mod +from evals.report import identity def _row(task_id: str = "R1", **overrides: Any) -> dict[str, Any]: @@ -424,3 +425,43 @@ def test_malformed_exact_run_expectation_is_refused_instead_of_treated_as_legacy assert captured.out == "" assert "invalid run expectation" in captured.err assert "expected_task_ids contains duplicates" in captured.err + + +def test_infra_error_rows_without_a_manifest_do_not_refuse_the_comparison(tmp_path: Path): + """A row that died in seeding never ran an agent, so it has no manifest to carry. + + The first live A/B was refused because six infra_seed rows (L2 and L5, identical on both + surfaces) had no tool_manifest_fingerprint — rows every statistic already excludes. Demanding + identity from a row that never reached the surface refuses sound comparisons. + """ + path = tmp_path / "rows.jsonl" + surface = { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + "tool_manifest_fingerprint": "fp-a", + "success": True, + } + seed_failure = { + "task_id": "L5", + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + "error_class": "infra_seed", + "error": "boom", + } + path.write_text( + json.dumps(surface) + "\n" + json.dumps(seed_failure) + "\n", + encoding="utf-8", + ) + report = identity.validate_persisted_identity([path]) + assert report.files[0].values[identity.TOOL_MANIFEST_FIELD] == "fp-a" From 74b437276c0d0d48653dedaa7a308a4f91da798e Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 08:28:19 +0530 Subject: [PATCH 50/93] Read the attachment list the shape the SDK returns it in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `work_items.attachments.list` returns a bare `list[WorkItemAttachment]`, not the paged envelope every other list endpoint returns, so reading `.results` raised AttributeError. L5 died in seeding on every repetition of every run — six rows lost across the two A/B batteries — and it read as an environment problem rather than a one-line type mistake. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/work_items.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 5bd7af1a..b24483db 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -389,7 +389,10 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, project_id=project_id, work_item_id=attachment_target_id, ) - confirmed_rows = list(attachments.results or []) + # attachments.list returns a bare list[WorkItemAttachment], not the paged envelope the + # other list endpoints return. Assuming `.results` raised AttributeError on every L5 + # repetition, which surfaced as infra_seed and cost the task its whole row budget. + confirmed_rows = list(attachments if isinstance(attachments, list) else (attachments.results or [])) for attachment in confirmed_rows: record_seeded_entity(context, "attachment", getattr(attachment, "id", None)) confirmed_attachment_count = len(confirmed_rows) @@ -416,6 +419,7 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st if not project_id or not work_item_id: missing = [name for name, value in (("project_id", project_id), ("work_item_id", work_item_id)) if not value] raise RuntimeError(f"seed L2 fixture error: missing {', '.join(missing)}") + page = plane.work_items.activities.list( workspace_slug=workspace_slug, project_id=project_id, @@ -423,6 +427,7 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st ) rows = page.results if hasattr(page, "results") else page activity_rows = list(rows or []) + candidates = [str(value) for value in context.get("l2_comment_phrases") or []] activity_count = len(activity_rows) if activity_count < 1: raise TaskSkipped("env:no-activity-worker") @@ -430,7 +435,6 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st if str(context.get("task_id") or "") == "L2": randomised = context.setdefault("randomized_truth", {}).setdefault("L2.activity_count", {}) randomised["confirmed"] = activity_count - candidates = [str(value) for value in context.get("l2_comment_phrases") or []] response_blob = _serialized_rows(activity_rows) visible = [value for value in candidates if value in response_blob] if not visible: From 5dbffca2db0533da8235906519345d1e08821f49 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 08:47:50 +0530 Subject: [PATCH 51/93] Ask L2 for evidence Plane can actually give MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L2's seeder required its randomized comment phrase to appear in the work item's activity readback. Plane's activity API never returns comment text — it returns the creation row and nothing else — so the requirement could not be satisfied by any agent, any worker, or any amount of waiting. L2 failed in seeding on every repetition and was absent from every result this harness has produced. Evidence is now the activity count bound to the seeded work item, which is what L2 asks for in its prompt and what its verifier already checks. The count comes from the same paginated envelope the agent reads, so provenance still requires a request that named the target and a response that carried the answer. CATALOG_REVISION 8: this changes what L2 measures. Nothing is lost across the transition, because before it L2 produced no rows at all. The pinned fingerprint and revision tests are updated rather than removed, and the gate test now encodes the new contract instead of the impossible one. Canary: 35/35 verified, 0 skipped, 0 errored — the whole catalog for the first time. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/work_items.py | 11 +++++------ evals/tasks/catalog.py | 7 ++++++- tests/evals/seed/test_read_randomization.py | 9 ++++++++- tests/evals/seed/test_seed.py | 14 +++++++++++--- tests/evals/tasks/test_catalog.py | 11 +++++++---- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index b24483db..694ffa5f 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -427,7 +427,6 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st ) rows = page.results if hasattr(page, "results") else page activity_rows = list(rows or []) - candidates = [str(value) for value in context.get("l2_comment_phrases") or []] activity_count = len(activity_rows) if activity_count < 1: raise TaskSkipped("env:no-activity-worker") @@ -435,8 +434,8 @@ def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[st if str(context.get("task_id") or "") == "L2": randomised = context.setdefault("randomized_truth", {}).setdefault("L2.activity_count", {}) randomised["confirmed"] = activity_count - response_blob = _serialized_rows(activity_rows) - visible = [value for value in candidates if value in response_blob] - if not visible: - raise RuntimeError("seed L2 fixture error: activity readback omitted randomized comment evidence") - set_target_evidence(context, visible, target_ids=[work_item_id]) + # Evidence is the activity count, which is what L2 actually asks for and what its + # verifier checks. It used to require the seeded comment phrase to appear in the + # activity readback — evidence Plane's activity API never emits: the endpoint returns + # the creation row and no comment text, so every L2 repetition died in seeding. + set_target_count_evidence(context, activity_count, target_ids=[work_item_id]) diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index cfcc4ced..911ef1ec 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,9 +80,14 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 7 +CATALOG_REVISION = 8 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. +Revision 8 binds L2's provenance to the activity count its verifier already checks, instead of +requiring the seeded comment phrase to appear in the activity readback. Plane's activity API +never emits comment text — it returns the creation row only — so the old requirement was +unsatisfiable and L2 failed in seeding on every repetition. L2 results are not comparable +across this transition, because before it there were none. Revision 7 makes read tasks require response evidence bound to the target entity and gives C2/R7 randomised, immutable seed-time oracles; pre-revision read results are not comparable. Revision 6 stops W8 and W9 asking for unverifiable logged-date and batching properties; diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index f6adb68b..88038c93 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -178,7 +178,14 @@ def test_work_item_read_truth_is_randomized_and_api_confirmed(task_id, oracle_ke assert ctx[oracle_key] not in (None, "", []) assert "confirmed" in ctx["randomized_truth"][truth_key] - assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + if task_id == "L2": + # L2 binds the activity count, not a sentinel value: Plane's activity payload carries + # the creation row and never the seeded comment text (revision 8). + assert ctx["evidence_aggregates"][TARGET_ENTITY_EVIDENCE] == ( + {"kind": "total_count", "value": ctx[oracle_key]}, + ) + else: + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index 9cbc26ae..48530034 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -1276,6 +1276,10 @@ def test_collision_category_coverage_matches_task_prompts(): id="successful-nonempty-read-proceeds", ), pytest.param( + # Revision 8: a non-empty read is sufficient, and evidence is the activity count. + # This case used to require the seeded comment phrase in the readback and expect a + # fixture error without it — a contract Plane's activity API can never satisfy, + # because it returns the creation row and never the comment text. { "task_id": "L2", "project_id": "p1", @@ -1283,9 +1287,9 @@ def test_collision_category_coverage_matches_task_prompts(): "l2_comment_phrases": ["hidden seeded comment"], }, [SimpleNamespace(id="a1", comment="unrelated activity")], - RuntimeError, - "fixture error: activity readback omitted randomized comment evidence", - id="nonempty-read-without-seeded-evidence-is-fixture-error", + None, + None, + id="nonempty-read-without-comment-text-binds-count-evidence", ), ], ) @@ -1300,6 +1304,10 @@ def list_activities(**kwargs): plane = SimpleNamespace(work_items=SimpleNamespace(activities=SimpleNamespace(list=list_activities))) if expected_error is None: _gate_activity_worker(plane, "ws", context) + if context.get("task_id") == "L2": + aggregates = context["evidence_aggregates"][TARGET_ENTITY_EVIDENCE] + assert aggregates == ({"kind": "total_count", "value": 1},) + assert context["evidence_targets"][TARGET_ENTITY_EVIDENCE] == ("wi-r5",) else: with pytest.raises(expected_error, match=match): _gate_activity_worker(plane, "ws", context) diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 076f1ac7..c7b6f4af 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -99,7 +99,7 @@ # "77230f96962d" at revision 6 before target-entity response evidence. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "e059523c9f3d" +PINNED_SYNTHETIC_BATTERY = "3f6a3f44455d" @pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) @@ -311,10 +311,13 @@ def test_fingerprint_records_the_revision_transition(): unverifiable logged-date ask and W9's unverifiable batching ask, and tightens the affected end-state verifiers; its full-catalog value was ``ccf39203f656``. Revision 7 binds read provenance to target-entity response evidence and gives C2/R7 randomised, - immutable seed-time oracles. Results across these transitions are not comparable. + immutable seed-time oracles; its full-catalog value was ``9ea76bf22ba0``. Revision 8 binds + L2's provenance to the activity count its verifier already checks, because the seeded + comment phrase it used to require is never present in Plane's activity payload. Results + across these transitions are not comparable. Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 7 - assert battery_fingerprint() == "9ea76bf22ba0" + assert CATALOG_REVISION == 8 + assert battery_fingerprint() == "61bddef0dc76" From f190185129ddf4611283a7fc58c1e0d6e194eadf Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 11:18:03 +0530 Subject: [PATCH 52/93] Let a count be evidence at the seed gate too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live runner refuses to start a read task whose seed registered no target-bound evidence. It asked `configured_evidence_labels` for sentinels and targets and left out aggregates, so the gate was stricter than the matcher it guards: the proxy matches an exact total_count for a targeted request exactly as it matches a sentinel value. A read task whose answer *is* a count could therefore register its evidence and still be rejected as having registered none. L2 failed that way on all three repetitions of the first battery to reach it — the seeding bug that had hidden the gate was fixed one commit earlier, so this only became reachable now. The canary never saw it: it does not run the live seed gate. The test covers the predicate directly, including that targets alone and an unbound aggregate still do not count. Co-Authored-By: Claude Opus 5 (1M context) --- evals/runner/live.py | 8 +++++++- tests/evals/test_evidence.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/evals/test_evidence.py diff --git a/evals/runner/live.py b/evals/runner/live.py index 0c489f3a..998d74c2 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -184,8 +184,14 @@ def _seed_fixtures( ctx=context, task_id=str(task["id"]), ) + # Aggregates count as registered evidence: the proxy matches an exact total_count for a + # targeted request exactly as it matches a sentinel value. Omitting them here made the + # gate stricter than the matcher, so a task whose answer *is* a count could not seed — + # L2 registered its activity count and was rejected as having registered nothing. if "read" in set(task.get("tags") or set()) and not configured_evidence_labels( - context.get("evidence_sentinels"), context.get("evidence_targets") + context.get("evidence_sentinels"), + context.get("evidence_targets"), + context.get("evidence_aggregates"), ): raise RuntimeError(f"{task['id']} seed did not register target-bound response evidence") except TaskSkipped as skip: diff --git a/tests/evals/test_evidence.py b/tests/evals/test_evidence.py new file mode 100644 index 00000000..ff14fc68 --- /dev/null +++ b/tests/evals/test_evidence.py @@ -0,0 +1,17 @@ +def test_aggregate_evidence_alone_counts_as_registered_target_bound_evidence(): + """A count is evidence: the proxy matches an exact total_count for a targeted request. + + The live runner's seed gate asked only for sentinels and targets, so a read task whose + answer *is* a count could register its evidence and still be rejected as having registered + none. L2 failed that way on every repetition of the first full-catalog battery, after the + seeding bug that had hidden it was fixed. + """ + from evals.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels + + targets = {TARGET_ENTITY_EVIDENCE: ("wi-1",)} + aggregates = {TARGET_ENTITY_EVIDENCE: ({"kind": "total_count", "value": 3},)} + + assert configured_evidence_labels(None, targets, aggregates) == (TARGET_ENTITY_EVIDENCE,) + # Targets alone are still not evidence, and neither is an aggregate with nothing to bind to. + assert configured_evidence_labels(None, targets, None) == () + assert configured_evidence_labels(None, None, aggregates) == () From 8da1d6ca1b8fd2f9ca4fe3625b778b63b892489f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 13:07:15 +0530 Subject: [PATCH 53/93] Bind R1 and I2 evidence to the state that carries the answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A work item's `state` is an id, so reading its name takes a second call: one names the work item and returns no name, the next returns the name and names only the state. Evidence bound to the work item alone demanded both halves in a single response, which no surface that returns state as an id can produce. R1 and I2 failed every repetition of the first full-catalog battery while answering correctly — the note read `answer_correct=true ... provenance=missing (0 evidence-bearing of 7 successful Plane calls)`. The consolidated surface returns `state` as a bare UUID, confirmed directly against v0.3.0, so the second call is the only way to the name. The seeded state is a target too. Both ids are seeded, so the agent must still name the thing it was asked about; it may now do so across the two calls the surface actually requires. Verified live: R1 and I2 both 0/3 -> 3/3. CATALOG_REVISION 9. Pinned fingerprint, revision and synthetic-battery values updated rather than removed. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/work_items.py | 12 +++++++++++- evals/tasks/catalog.py | 6 +++++- tests/evals/tasks/test_catalog.py | 11 +++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 694ffa5f..650ba5ff 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -295,7 +295,17 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, oracle_key = "r1_state_name" if task_id == "R1" else "i2_state_name" context[oracle_key] = confirmed_name context["randomized_truth"][f"{task_id}.state"]["confirmed"] = confirmed_name - set_target_evidence(context, [confirmed_name], target_ids=[target_id]) + # The state entity carries the answer, so it is a target too. A work item's `state` is a + # bare id, so reading the name takes a second call — one naming the work item and + # returning no name, one returning the name and naming only the state. Binding evidence + # to the work item alone demanded both halves in a single response, which no surface that + # returns state as an id can produce: R1 and I2 failed every repetition while answering + # correctly. Both ids are seeded, so the agent must still name the thing it was asked about. + target_ids = [target_id] + state_id = _as_id(detail.state) + if state_id: + target_ids.append(state_id) + set_target_evidence(context, [confirmed_name], target_ids=target_ids) if task_id == "R2": confirmed_titles = _confirm_open_urgent_items(plane, workspace_slug, project_id) diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 911ef1ec..3f976294 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,9 +80,13 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 8 +CATALOG_REVISION = 9 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. +Revision 9 also binds R1/I2 provenance to the seeded state, not the work item alone. A work +item's `state` is an id, so resolving its name takes a second call, and the old rule needed the +target id and the answer in one response — unsatisfiable against a surface that does not expand +state. Both tasks failed every repetition while answering correctly. Revision 8 binds L2's provenance to the activity count its verifier already checks, instead of requiring the seeded comment phrase to appear in the activity readback. Plane's activity API never emits comment text — it returns the creation row only — so the old requirement was diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index c7b6f4af..238da366 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -99,7 +99,7 @@ # "77230f96962d" at revision 6 before target-entity response evidence. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "3f6a3f44455d" +PINNED_SYNTHETIC_BATTERY = "ea5bdba36109" @pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) @@ -313,11 +313,14 @@ def test_fingerprint_records_the_revision_transition(): binds read provenance to target-entity response evidence and gives C2/R7 randomised, immutable seed-time oracles; its full-catalog value was ``9ea76bf22ba0``. Revision 8 binds L2's provenance to the activity count its verifier already checks, because the seeded - comment phrase it used to require is never present in Plane's activity payload. Results + comment phrase it used to require is never present in Plane's activity payload; its + full-catalog value was ``61bddef0dc76``. Revision 9 also binds R1/I2 provenance to the seeded + state, because a work item's state is an id and resolving its name takes a second call. + Results across these transitions are not comparable. Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 8 - assert battery_fingerprint() == "61bddef0dc76" + assert CATALOG_REVISION == 9 + assert battery_fingerprint() == "e9134604a0a7" From 181840cbf0aec694dbd090cfdc9feadc4b0d880a Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 13:19:58 +0530 Subject: [PATCH 54/93] Let R6 be proven by a count per project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R6 registered only one provable shape: a single count grouped by project_id. Counting each project separately reaches the same answer and is what every agent actually did, so a correct winner scored as unproven on all three repetitions. Both aggregate setters replaced the label's registered specs instead of appending, so a task could only ever have one acceptable shape — which is why the two could not simply be called together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/evidence.py | 53 ++++++++++++++++++++----------- evals/seed/projects.py | 6 +++- evals/tasks/catalog.py | 6 +++- tests/evals/tasks/test_catalog.py | 12 ++++--- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/evals/evidence.py b/evals/evidence.py index 263d0be1..5e8df739 100644 --- a/evals/evidence.py +++ b/evals/evidence.py @@ -421,6 +421,31 @@ def observed_aggregate_labels(observations: Any, aggregates: Any) -> list[str]: return sorted(matched) +def _register_targets(context: dict[str, Any], target_ids: Sequence[Any], *, what: str) -> None: + """Add seeded entity IDs to the label's target set, keeping any already registered.""" + clean = tuple(dict.fromkeys(str(value).strip() for value in target_ids if value is not None and str(value).strip())) + if not clean: + raise RuntimeError(f"{what} has no seeded target entity ids") + targets = context.setdefault("evidence_targets", {}) + current = targets.get(TARGET_ENTITY_EVIDENCE, ()) + targets[TARGET_ENTITY_EVIDENCE] = tuple(dict.fromkeys((*current, *clean))) + + +def _add_aggregate_specs(context: dict[str, Any], specs: Sequence[dict[str, Any]]) -> None: + """Append acceptable aggregate shapes rather than replacing the registered ones. + + A task may reach its answer by more than one honest call shape — R6's winner is + provable by two per-project counts or by one count grouped by project. Replacing + here privileged whichever seeder ran last, and scored every other path unproven. + """ + aggregates = context.setdefault("evidence_aggregates", {}) + registered = list(aggregates.get(TARGET_ENTITY_EVIDENCE, ())) + for spec in specs: + if spec not in registered: + registered.append(spec) + aggregates[TARGET_ENTITY_EVIDENCE] = tuple(registered) + + def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, target_ids: Sequence[Any]) -> None: """Register API-confirmed values and the entity IDs whose reads may prove them.""" clean_values: list[str] = [] @@ -433,24 +458,16 @@ def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, targe clean = tuple(dict.fromkeys(clean_values)) if not clean: raise RuntimeError("target evidence has no API-confirmed sentinel values") - clean_targets = tuple( - dict.fromkeys(str(value).strip() for value in target_ids if value is not None and str(value).strip()) - ) - if not clean_targets: - raise RuntimeError("target evidence has no seeded target entity ids") + _register_targets(context, target_ids, what="target evidence") context["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: clean} - context["evidence_targets"] = {TARGET_ENTITY_EVIDENCE: clean_targets} -def set_target_count_evidence(context: dict[str, Any], count: int, *, target_ids: Sequence[Any]) -> None: - """Allow an exact ``total_count`` response whose request names the seeded target.""" - clean_targets = tuple(str(value).strip() for value in target_ids if value is not None and str(value).strip()) - if not clean_targets: - raise RuntimeError("target count evidence has no seeded target entity ids") - context.setdefault("evidence_targets", {})[TARGET_ENTITY_EVIDENCE] = clean_targets - context.setdefault("evidence_aggregates", {})[TARGET_ENTITY_EVIDENCE] = ( - {"kind": "total_count", "value": int(count)}, - ) +def set_target_count_evidence(context: dict[str, Any], *counts: int, target_ids: Sequence[Any]) -> None: + """Allow an exact ``total_count`` response whose request names a seeded target.""" + if not counts: + raise RuntimeError("target count evidence has no API-confirmed counts") + _register_targets(context, target_ids, what="target count evidence") + _add_aggregate_specs(context, [{"kind": "total_count", "value": int(count)} for count in counts]) def set_target_grouped_count_evidence(context: dict[str, Any], values: Mapping[Any, int]) -> None: @@ -458,10 +475,8 @@ def set_target_grouped_count_evidence(context: dict[str, Any], values: Mapping[A clean = {str(target): int(count) for target, count in values.items() if str(target).strip()} if not clean: raise RuntimeError("target grouped-count evidence has no seeded targets") - context.setdefault("evidence_targets", {})[TARGET_ENTITY_EVIDENCE] = tuple(clean) - context.setdefault("evidence_aggregates", {})[TARGET_ENTITY_EVIDENCE] = ( - {"kind": "grouped_counts", "values": clean}, - ) + _register_targets(context, clean, what="target grouped-count evidence") + _add_aggregate_specs(context, [{"kind": "grouped_counts", "values": clean}]) __all__ = [ diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 9cb29dd4..40a7a252 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -14,7 +14,7 @@ from plane.models.workspaces import WorkspaceFeature from evals.errors import TaskSkipped -from evals.evidence import set_target_evidence, set_target_grouped_count_evidence +from evals.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth @@ -337,7 +337,11 @@ def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: "winner": context["r6_more_bugs_project"], } set_target_evidence(context, [*main_titles, *second_titles], target_ids=[main_id, project.id]) + # Two honest call shapes reach this answer: one count grouped by project_id, or one + # count per project. Only the grouped shape was provable, so every agent that took + # the per-project route answered correctly and scored as unproven. set_target_grouped_count_evidence( context, {main_id: confirmed_main, str(project.id): confirmed_second}, ) + set_target_count_evidence(context, confirmed_main, confirmed_second, target_ids=[main_id, project.id]) diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 3f976294..1d45b9c0 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,9 +80,13 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 9 +CATALOG_REVISION = 10 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. +Revision 10 lets R6's provenance be proven by one count per project, not only by a single +count grouped by project. Both are honest routes to the winner, but only the grouped one +was registered, so every agent that counted each project separately answered correctly and +scored as unproven. R6 results are not comparable across this transition. Revision 9 also binds R1/I2 provenance to the seeded state, not the work item alone. A work item's `state` is an id, so resolving its name takes a second call, and the old rule needed the target id and the answer in one response — unsatisfiable against a surface that does not expand diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 238da366..37bd302d 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -97,9 +97,10 @@ # "9f2c2feb2e24" at revision 4 before read provenance and randomised truth. # "7b8dc6bd2f8f" at revision 5 before unverifiable W8/W9 asks were removed. # "77230f96962d" at revision 6 before target-entity response evidence. +# "ea5bdba36109" at revision 9 before R6 accepted per-project counts as provenance. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "ea5bdba36109" +PINNED_SYNTHETIC_BATTERY = "05176e65e594" @pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) @@ -315,12 +316,13 @@ def test_fingerprint_records_the_revision_transition(): L2's provenance to the activity count its verifier already checks, because the seeded comment phrase it used to require is never present in Plane's activity payload; its full-catalog value was ``61bddef0dc76``. Revision 9 also binds R1/I2 provenance to the seeded - state, because a work item's state is an id and resolving its name takes a second call. - Results + state, because a work item's state is an id and resolving its name takes a second call; + its full-catalog value was ``e9134604a0a7``. Revision 10 accepts one count per project as + R6 provenance, alongside the single count grouped by project it already accepted. Results across these transitions are not comparable. Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 9 - assert battery_fingerprint() == "e9134604a0a7" + assert CATALOG_REVISION == 10 + assert battery_fingerprint() == "9cce7ce77310" From 275683a9944e61b4ee56ecda24a855a4d5be9e17 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 14:10:51 +0530 Subject: [PATCH 55/93] Make provenance a property instead of a list of accepted routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sentinel is a per-run random string that exists only inside Plane, and the agent's only route to Plane is the surface being measured. Its presence in a response the agent received is therefore proof on its own, so the request no longer has to name a particular entity. That target binding was an enumeration of approved routes through 183 actions, and it could never be complete. It rejected reading a state by listing a project's states, finding the active cycle by listing a project's cycles, and counting two projects separately — six defects in one week, every one a correct answer scored as unproven. Counts keep the binding, because a small integer is guessable where a random string is not. DESIGN.md now states both rules, and states that the harness measures a cooperative agent rather than an adversarial one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/DESIGN.md | 58 +++++++++++---- evals/README.md | 22 ++++-- evals/drivers/driver.py | 9 +-- evals/evidence.py | 82 ++++++++++----------- evals/proxy.py | 9 +-- evals/seed/cycles.py | 6 +- evals/seed/projects.py | 2 +- evals/seed/releases.py | 2 +- evals/seed/states.py | 2 +- evals/seed/work_items.py | 22 ++---- evals/tasks/catalog.py | 13 +++- tests/evals/drivers/test_api_driver.py | 7 +- tests/evals/seed/test_read_randomization.py | 12 +-- tests/evals/test_proxy.py | 8 +- 14 files changed, 132 insertions(+), 122 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 21ff2957..039b5a11 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -132,22 +132,50 @@ Payload recording is off by default because tool results contain live workspace make sidecars larger. The character-derived estimate remains useful for surface comparison because it is deterministic and monotonic in the recorded response size. -Read-task provenance does not turn payload recording back on. Each read seeder registers a -hidden, per-run sentinel and its seeded target entity ID. At the API loop or CLI recording -proxy, the harness requires the request arguments to target that ID and matches the sentinel -while the successful response is in memory, persisting only a non-sensitive -`observed_sentinels` label. CLI proxies receive only target IDs plus sentinel lengths and -SHA-256 fingerprints through a mode-0600 run-scoped file outside the agent cwd; the raw value +### Provenance: what counts as proof the answer came from the surface + +A read verifier asks two independent questions — is the answer right, and did the agent get +it from the tool surface. This is the definition of the second one. It is a property, not a +list of approved call sequences: an enumeration of routes through 183 actions can never be +complete, and each gap in it fails an agent that answered correctly by an unlisted route. + +**A sentinel proves itself.** A sentinel is a per-run random string a seeder wrote into +Plane — a state name, a work item title, a comment phrase. It exists nowhere else, and the +agent's only route to Plane is the surface under measurement. So if a sentinel appears in a +response the agent received, the agent used the surface. Nothing further is required: not +which entity the request named, not which tool was called, not how many calls it took. + +**A count does not.** Where the seeded truth is a number, presence proves nothing — a small +integer appears by coincidence. Aggregate evidence therefore keeps a target binding: an exact +`total_count` counts only from a request whose arguments name a seeded entity. R2 binds one +project's count; L2 binds a work item's activity count; R6 accepts either one count per +project or one count grouped by project, since both are honest routes to its answer. + +Matching happens while the successful response is in memory, and only a non-sensitive +`observed_sentinels` label is persisted. Read-task provenance does not turn payload recording +back on. CLI proxies receive sentinel lengths and SHA-256 fingerprints, plus the target IDs the +aggregate rule needs, through a mode-0600 run-scoped file outside the agent cwd; the raw value is absent even from that file. Every MCP proxy session reads the same file, and the driver -removes it with its temporary directory after the run. A successful response from -an unrelated entity therefore does not count, and neither the response body nor sentinel -enters the payload-free result row. Unavailable or incomplete matching is diagnosed and -cannot pass a read verifier. - -Two count-oriented tasks have an equally narrow alternative evidence shape. R2 accepts an -exact `total_count` only when the request arguments contain the seeded project ID; R6 accepts -only grouped counts containing both seeded project IDs with their exact API-confirmed counts. -No other read task receives this relaxation, and both transports still persist labels only. +removes it with its temporary directory after the run. Neither the response body nor the +sentinel enters the result row. Unavailable or incomplete matching is diagnosed and cannot +pass a read verifier. + +### Threat model: a cooperative agent + +The harness measures an agent that is trying to do the task, not one trying to defeat the +measurement. That is a deliberate scope decision, and the numbers should be read with it in +mind. + +The agent under measurement runs as the same OS user, in the same filesystem, holding the same +Plane credentials as the harness. An agent that wanted to could read the evidence +configuration, edit an earlier row in the result file, fabricate a proxy sidecar session, or +skip MCP and call Plane directly — and off-surface indicators report such signals without +gating success. None of this is closed by any verifier rule; closing it requires running the +agent in a separate trust domain, which is a different project. + +For the question this harness exists to answer — how well does our own tool surface serve a +capable agent — a cooperative agent is the right subject. Treat every integrity property above +as holding against accident and drift, not against an adversary. Provider usage is a different measurement: where the driver supplies it, the harness keeps input, output, cache-read, and cache-creation usage. Tool-result sizing describes one source diff --git a/evals/README.md b/evals/README.md index d02f2fc8..5f895aa2 100644 --- a/evals/README.md +++ b/evals/README.md @@ -118,14 +118,20 @@ estimate is monotonic in the thing being compared anyway. Do not enable payload habit; use it only when the more sensitive, larger sidecar is justified. Read-task provenance is stricter than “a call happened.” Seeders place a hidden per-run -sentinel on the target entity, and the API driver or CLI proxy records only whether a -successful response exposed it **and its request targeted that seeded entity ID**. CLI -proxies receive only target IDs and one-way value fingerprints through a private run-scoped -file; the raw sentinel is absent even if that file is inspected. Every proxy session can read it, and the driver -removes it with its temporary directory after the run. Result rows contain the matched label, never the sentinel -value or response body. Thus an unrelated -successful response cannot satisfy provenance; unavailable or incomplete matching is -diagnosed and fails closed. +sentinel — a random string that exists only inside Plane — and the API driver or CLI proxy +records whether a successful response exposed it. Because the agent's only route to Plane is +the surface under measurement, a sentinel in a response it received is proof of surface use by +itself; the harness deliberately does not also require the request to have named a particular +entity, because that rejected honest routes to the same answer. Where the seeded truth is a +count rather than a string, presence proves nothing and the target binding still applies: an +exact `total_count` counts only from a request naming a seeded entity. `evals/DESIGN.md` states +both rules and the threat model they hold under. + +CLI proxies receive one-way value fingerprints, plus the target IDs the count rule needs, +through a private run-scoped file; the raw sentinel is absent even if that file is inspected. +Every proxy session can read it, and the driver removes it with its temporary directory after +the run. Result rows contain the matched label, never the sentinel value or response body. +Unavailable or incomplete matching is diagnosed and fails closed. ### Reading results diff --git a/evals/drivers/driver.py b/evals/drivers/driver.py index a58d755b..9200f1a2 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/driver.py @@ -369,14 +369,7 @@ async def _run_task( ) calls[idx]["observed_aggregates"] = aggregate_observations calls[idx]["observed_sentinels"] = sorted( - set( - observed_sentinel_labels( - result.text, - evidence, - request_args=calls[idx]["args"], - evidence_targets=targets, - ) - ) + set(observed_sentinel_labels(result.text, evidence)) | set(observed_aggregate_labels(aggregate_observations, aggregates)) ) pending_results.append((idx, result.text)) diff --git a/evals/evidence.py b/evals/evidence.py index 5e8df739..a7e0a66b 100644 --- a/evals/evidence.py +++ b/evals/evidence.py @@ -1,10 +1,19 @@ -"""Target-bound response-evidence matching without retaining response bodies. +"""Response-evidence matching without retaining response bodies. -Seeders register hidden, per-run sentinel values under a non-sensitive label. Drivers -compare each Plane response with those values only when the request targets the seeded -entity, then retain only the labels that matched. CLI proxies consume the matching -configuration from a one-shot file before the agent starts; sentinel values never enter -agent-visible argv/config, result rows, or payload-free sidecars. +Provenance asks one question: did the answer come from the tool surface? Two kinds of +evidence answer it, under different rules, because they differ in how guessable they are. + +A **sentinel** is a per-run random string a seeder wrote into Plane. The agent's only +route to Plane is the surface, so the string appearing in a response the agent received +proves surface use by itself. Nothing else needs to hold. + +An **aggregate** is a count. A small integer is guessable, so a count proves nothing on +its own: it counts only from a request that named a seeded entity. + +Drivers compare each Plane response against the configured evidence and retain only the +labels that matched. CLI proxies consume the matching configuration from a file before the +agent starts; sentinel values never enter agent-visible argv/config, result rows, or +payload-free sidecars. """ from __future__ import annotations @@ -115,11 +124,16 @@ def normalize_evidence_aggregate_shapes(value: Any) -> dict[str, tuple[dict[str, def configured_evidence_labels(sentinels: Any, targets: Any, aggregates: Any = None) -> tuple[str, ...]: - """Return labels that have both response values and target entity IDs.""" - values_by_label = normalize_evidence_sentinels(sentinels) + """Return labels this run can actually prove, by whichever rule governs their kind. + + A sentinel proves itself. A count does not — a small integer is guessable, so it + counts only from a request that named a seeded entity, and an aggregate label with + no registered target can never match. + """ + sentinel_labels = normalize_evidence_sentinels(sentinels).keys() targets_by_label = normalize_evidence_targets(targets) - aggregate_labels = normalize_evidence_aggregates(aggregates) - return tuple(sorted((values_by_label.keys() | aggregate_labels.keys()) & targets_by_label.keys())) + aggregate_labels = normalize_evidence_aggregates(aggregates).keys() & targets_by_label.keys() + return tuple(sorted(sentinel_labels | aggregate_labels)) def fingerprint_evidence_sentinels(value: Any) -> dict[str, tuple[tuple[int, str], ...]]: @@ -268,46 +282,27 @@ def contains(value: Any) -> bool: return contains(request_args) -def observed_sentinel_labels( - response_text: str, - sentinels: Any, - *, - request_args: Any, - evidence_targets: Any, -) -> list[str]: - """Return labels whose target request exposed a hidden value in its response.""" +def observed_sentinel_labels(response_text: str, sentinels: Any) -> list[str]: + """Return labels whose hidden value appeared in this response. + + A sentinel is a per-run random string written into Plane at seed time, and the + agent's only route to Plane is the tool surface. Its presence in a response the + agent received is therefore proof of surface use on its own, with no need to also + inspect which entity the request named. + """ text = str(response_text or "") if not text: return [] normalized = normalize_evidence_sentinels(sentinels) - targets = normalize_evidence_targets(evidence_targets) - return sorted( - label - for label, values in normalized.items() - if label in targets - and _request_targets(request_args, targets[label]) - and any(value in text for value in values) - ) + return sorted(label for label, values in normalized.items() if any(value in text for value in values)) -def observed_fingerprint_labels( - response_text: str, - fingerprints: Any, - *, - request_args: Any, - evidence_targets: Any, -) -> list[str]: - """Match target-bound value fingerprints without ever receiving the raw values.""" +def observed_fingerprint_labels(response_text: str, fingerprints: Any) -> list[str]: + """Match sentinel fingerprints without ever receiving the raw values.""" text = str(response_text or "") if not text: return [] - normalized = normalize_evidence_fingerprints(fingerprints) - targets = normalize_evidence_targets(evidence_targets) - eligible = { - label: specs - for label, specs in normalized.items() - if label in targets and _request_targets(request_args, targets[label]) - } + eligible = normalize_evidence_fingerprints(fingerprints) if not eligible: return [] @@ -446,8 +441,8 @@ def _add_aggregate_specs(context: dict[str, Any], specs: Sequence[dict[str, Any] aggregates[TARGET_ENTITY_EVIDENCE] = tuple(registered) -def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, target_ids: Sequence[Any]) -> None: - """Register API-confirmed values and the entity IDs whose reads may prove them.""" +def set_target_evidence(context: dict[str, Any], values: Sequence[Any]) -> None: + """Register the API-confirmed hidden values whose presence proves surface use.""" clean_values: list[str] = [] for value in values: if value is None: @@ -458,7 +453,6 @@ def set_target_evidence(context: dict[str, Any], values: Sequence[Any], *, targe clean = tuple(dict.fromkeys(clean_values)) if not clean: raise RuntimeError("target evidence has no API-confirmed sentinel values") - _register_targets(context, target_ids, what="target evidence") context["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: clean} diff --git a/evals/proxy.py b/evals/proxy.py index 2a46e893..4d976b48 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -162,7 +162,7 @@ def __init__( self.evidence_targets = normalize_evidence_targets(evidence_targets) self.evidence_aggregates = normalize_evidence_aggregate_shapes(evidence_aggregates) self.evidence_active = bool( - (self.evidence_fingerprints.keys() | self.evidence_aggregates.keys()) & self.evidence_targets.keys() + self.evidence_fingerprints or (self.evidence_aggregates.keys() & self.evidence_targets.keys()) ) self._lock = threading.Lock() self._error_lock = threading.Lock() @@ -316,12 +316,7 @@ def on_server_message(self, obj: dict[str, Any]) -> None: # target-bound aggregate values the agent already received. The # expected aggregate truth and complete result body never enter # the proxy process. - row["observed_sentinels"] = observed_fingerprint_labels( - result_text, - self.evidence_fingerprints, - request_args=pending["args"], - evidence_targets=self.evidence_targets, - ) + row["observed_sentinels"] = observed_fingerprint_labels(result_text, self.evidence_fingerprints) row["observed_aggregates"] = observed_aggregates( result_text, self.evidence_aggregates, diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py index 16f171e1..aff2e7d3 100644 --- a/evals/seed/cycles.py +++ b/evals/seed/cycles.py @@ -199,8 +199,4 @@ def seed_cycles( "active_titles": list(confirmed_active_titles), "overdue_titles": list(confirmed_overdue_titles), } - set_target_evidence( - context, - [context["r4_cycle_name"], *confirmed_active_titles], - target_ids=[current.id], - ) + set_target_evidence(context, [context["r4_cycle_name"], *confirmed_active_titles]) diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 40a7a252..0394d960 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -336,7 +336,7 @@ def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: "second": confirmed_second, "winner": context["r6_more_bugs_project"], } - set_target_evidence(context, [*main_titles, *second_titles], target_ids=[main_id, project.id]) + set_target_evidence(context, [*main_titles, *second_titles]) # Two honest call shapes reach this answer: one count grouped by project_id, or one # count per project. Only the grouped shape was provable, so every agent that took # the per-project route answered correctly and scored as unproven. diff --git a/evals/seed/releases.py b/evals/seed/releases.py index 8470d8b5..4a6039c4 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -83,4 +83,4 @@ def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any "changelog": confirmed_text, "items": list(items), } - set_target_evidence(context, items, target_ids=[release.id]) + set_target_evidence(context, items) diff --git a/evals/seed/states.py b/evals/seed/states.py index 355bf69c..c840e9d4 100644 --- a/evals/seed/states.py +++ b/evals/seed/states.py @@ -53,7 +53,7 @@ def seed_r7_state_oracle(plane: PlaneClient, workspace_slug: str, context: dict[ "confirmed": list(pairs), }, ) - set_target_evidence(context, [state_name], target_ids=[project_id]) + set_target_evidence(context, [state_name]) __all__ = ["seed_r7_state_oracle"] diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 650ba5ff..263b1e4e 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -295,17 +295,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, oracle_key = "r1_state_name" if task_id == "R1" else "i2_state_name" context[oracle_key] = confirmed_name context["randomized_truth"][f"{task_id}.state"]["confirmed"] = confirmed_name - # The state entity carries the answer, so it is a target too. A work item's `state` is a - # bare id, so reading the name takes a second call — one naming the work item and - # returning no name, one returning the name and naming only the state. Binding evidence - # to the work item alone demanded both halves in a single response, which no surface that - # returns state as an id can produce: R1 and I2 failed every repetition while answering - # correctly. Both ids are seeded, so the agent must still name the thing it was asked about. - target_ids = [target_id] - state_id = _as_id(detail.state) - if state_id: - target_ids.append(state_id) - set_target_evidence(context, [confirmed_name], target_ids=target_ids) + set_target_evidence(context, [confirmed_name]) if task_id == "R2": confirmed_titles = _confirm_open_urgent_items(plane, workspace_slug, project_id) @@ -313,7 +303,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, raise RuntimeError("seed R2: API readback found no urgent open work items") context["r2_urgent_open_count"] = len(confirmed_titles) context["randomized_truth"]["R2.urgent_open_count"]["confirmed"] = len(confirmed_titles) - set_target_evidence(context, confirmed_titles, target_ids=[project_id]) + set_target_evidence(context, confirmed_titles) set_target_count_evidence(context, len(confirmed_titles), target_ids=[project_id]) if task_id == "R3": @@ -341,7 +331,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, "count": len(confirmed_due_titles), }, } - set_target_evidence(context, confirmed_due_titles, target_ids=[project_id]) + set_target_evidence(context, confirmed_due_titles) if task_id == "R5": page = plane.work_items.comments.list( @@ -361,7 +351,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, ) context["r5_comment_phrases"] = confirmed_comments context["randomized_truth"]["R5.comments"]["confirmed"] = list(confirmed_comments) - set_target_evidence(context, confirmed_comments, target_ids=[target_id]) + set_target_evidence(context, confirmed_comments) if task_id == "L1": work_item_id = str(context["fixture_item_ids"].get(PAYMENT_WEBHOOK_TITLE) or "") @@ -376,7 +366,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, if confirmed_id != work_item_id: raise RuntimeError(f"seed L1: target work item readback id={confirmed_id!r}; want {work_item_id!r}") context["l1_expected_summary_ids"] = [confirmed_id] - set_target_evidence(context, [confirmed_id], target_ids=[project_id]) + set_target_evidence(context, [confirmed_id]) if task_id == "L5": attachment_target_id = context["fixture_item_ids"][PAYMENT_WEBHOOK_TITLE] @@ -414,7 +404,7 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, raise RuntimeError( f"seed L5: attachment readback exposed {len(confirmed_names)} randomized names; want {attachment_count}" ) - set_target_evidence(context, confirmed_names, target_ids=[attachment_target_id]) + set_target_evidence(context, confirmed_names) def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 1d45b9c0..dc306efc 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -83,10 +83,15 @@ def task_author(task: dict[str, Any]) -> str: CATALOG_REVISION = 10 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. -Revision 10 lets R6's provenance be proven by one count per project, not only by a single -count grouped by project. Both are honest routes to the winner, but only the grouped one -was registered, so every agent that counted each project separately answered correctly and -scored as unproven. R6 results are not comparable across this transition. +Revision 10 replaces the per-task list of accepted provenance shapes with one rule per kind of +evidence. A sentinel is a per-run random string that exists only inside Plane, so its presence +in a response the agent received proves surface use on its own; the request no longer has to +name a particular entity. A count is guessable, so it still counts only from a request naming a +seeded entity, and R6 accepts one count per project as well as one count grouped by project. +The old rule enumerated routes through a 183-action surface and could never be complete: it +rejected reading a state by listing a project's states, finding a cycle by listing a project's +cycles, and counting two projects separately — all correct answers scored as unproven. Every +read task's results are not comparable across this transition. Revision 9 also binds R1/I2 provenance to the seeded state, not the work item alone. A work item's `state` is an id, so resolving its name takes a second call, and the old rule needed the target id and the answer in one response — unsatisfiable against a surface that does not expand diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 2165b48c..679fe523 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -466,8 +466,6 @@ def _api_driver_records_only_matching_evidence_labels(): ) session = FakeMcpSession( [ - # A real response carrying the sentinel is insufficient when the request - # targeted an unrelated entity (write-there/read-back bypass). ToolResult(call_id="a", text=f"state={sentinel}"), ToolResult(call_id="b", text=f"state={sentinel}"), ] @@ -476,11 +474,12 @@ def _api_driver_records_only_matching_evidence_labels(): run = run_driver( make_driver(backend, session), evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, - evidence_targets={TARGET_ENTITY_EVIDENCE: ["target"]}, ) + # The sentinel only exists inside Plane, so either response having it is proof of + # surface use; which entity the request named does not change that. assert run.evidence_trace_available is True - assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert run.calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert "result_text" not in run.calls[1] diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index 88038c93..1093030d 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -7,7 +7,7 @@ import pytest -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels from evals.seed.cycles import seed_cycles from evals.seed.states import seed_r7_state_oracle from evals.seed.work_items import require_activities, seed_work_items @@ -186,7 +186,12 @@ def test_work_item_read_truth_is_randomized_and_api_confirmed(task_id, oracle_ke ) else: assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] - assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] + # Whichever kind of evidence the task registered, the live seed gate must accept it. + assert configured_evidence_labels( + ctx.get("evidence_sentinels"), + ctx.get("evidence_targets"), + ctx.get("evidence_aggregates"), + ) == (TARGET_ENTITY_EVIDENCE,) def test_r2_randomized_counts_differ_between_rows_after_api_readback(): @@ -221,7 +226,6 @@ def test_r4_cycle_inventory_is_randomized_and_api_confirmed(): "overdue_titles": ctx["r4_overdue_titles"], } assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] - assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): @@ -237,7 +241,6 @@ def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): truth = ctx["randomized_truth"]["R7.states"] assert truth["confirmed"] == ctx["r7_state_pairs"] assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] - assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] def test_l1_seed_oracle_is_the_api_confirmed_target_id(): @@ -248,4 +251,3 @@ def test_l1_seed_oracle_is_the_api_confirmed_target_id(): assert ctx["l1_expected_summary_ids"] == [ctx["fixture_item_ids"]["Payment webhook drops retries"]] assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == tuple(ctx["l1_expected_summary_ids"]) - assert ctx["evidence_targets"][TARGET_ENTITY_EVIDENCE] == (ctx["project_id"],) diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index ad44e6c7..d8f4d30b 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -710,7 +710,6 @@ def _sidecar_recorder_unit(tmp_path): rec = SidecarRecorder( tmp_path / "a.jsonl", evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, - evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, ) rec.on_client_message( { @@ -753,9 +752,12 @@ def _sidecar_recorder_unit(tmp_path): assert calls[0]["origin"] == "plane" assert "result_text" not in calls[0] assert all("result_text" not in row for row in raw_calls) - assert calls[0]["observed_sentinels"] == [] + # A sentinel is a per-run random string that exists only inside Plane, so its + # presence proves the response came from the surface whichever entity the request + # named. Both calls received it; both are evidence. + assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] - assert raw_calls[0]["observed_sentinels"] == [] + assert raw_calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert raw_calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert sentinel not in (tmp_path / "a.jsonl").read_text(encoding="utf-8") assert rec.finalized is True From e980b699b3a112745af251a1b8160da05417fcf9 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 14:16:49 +0530 Subject: [PATCH 56/93] Stop charging infrastructure failures to the model A nonzero codex exit was scored as a finished attempt: the default stop reason was end_turn whatever the process did, so an authentication or network failure that still emitted a partial message got verified and counted in the success rate. opencode and antigravity already map a nonzero exit to error, so this skewed driver comparisons against them too. A plan-gated skip ends before the agent starts and so carries no tool manifest, exactly like a seed failure. Identity validation excluded only the latter, so one gated task made every offline report command exit 2 on a file the live summary called complete. Also: --dry-run described a project W11 does not build, the CLI listing test covered 24 of 35 tasks, and one usage branch did nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/README.md | 3 ++- evals/drivers/cli/codex.py | 11 ++++++++- evals/report/identity.py | 6 ++--- evals/report/load.py | 11 +++++++++ evals/results.py | 2 -- evals/seed/plan.py | 2 ++ tests/evals/drivers/test_cli_driver.py | 33 ++++++++++++++++++++++++++ tests/evals/report/test_identity.py | 24 +++++++++++++++++++ tests/evals/test_cli.py | 8 +++++-- 9 files changed, 91 insertions(+), 9 deletions(-) diff --git a/evals/README.md b/evals/README.md index 5f895aa2..adf634c7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -234,7 +234,8 @@ A task is a dict: `needs` tokens: `items`, `labels`, `bug_type`, `cycles`, `cycles_open_past`, `module`, `intake`, `customer`, `release`, `activity_feed`, `second_project`, -`leave_cycles_worklogs_off`. Each task gets its own freshly seeded project, so fixture +`leave_cycles_worklogs_off` (S5: cycles + worklogs + workspace customers off), +`leave_worklogs_off` (W11: worklogs only). Each task gets its own freshly seeded project, so fixture variants (e.g. `cycles_open_past`) don't leak between tasks. ### Writing a verifier diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index 51e4b688..f5651a08 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -424,13 +424,22 @@ def parse_output( } raw_ref = f"session:{session_id}" if session_id else None + # A nonzero exit means the process failed — authentication, network, a crash — so the + # transcript is not a finished attempt and must not be scored as one. opencode and + # antigravity already say so; codex defaulting to end_turn charged its own process + # failures to the model's success rate and biased every driver comparison against + # the other two. + stopped_reason = parsed.get("stopped_reason") or "end_turn" + if proc.returncode: + notes.append(f"codex_exit={proc.returncode}") + stopped_reason = "error" return CliOutput( calls=calls, final_text=parsed.get("final_text") or "", client_tool_calls=client_calls, usage=usage, usage_total=usage_total, - stopped_reason=parsed.get("stopped_reason") or "end_turn", + stopped_reason=stopped_reason, raw_ref=raw_ref, call_source=call_source, hit_max_turns=False, # codex exec has no max-turns flag in --help diff --git a/evals/report/identity.py b/evals/report/identity.py index 7813be85..20354949 100644 --- a/evals/report/identity.py +++ b/evals/report/identity.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -from evals.report.load import is_infra_error_row +from evals.report.load import is_unlaunched_row MISSING = "" IDENTITY_FIELDS = ("battery", "resolved_model", "provider", "driver", "server") @@ -122,11 +122,11 @@ def _validate_file(path: Path) -> tuple[FileIdentity, list[str]]: source = row_values or header_values or {MISSING: []} values[field] = next(iter(source)) - # A row that failed in seeding never launched an agent, so no tools/list was ever + # A row that ended during seeding never launched an agent, so no tools/list was ever # observed and there is no manifest to record. Demanding one from those rows refuses # comparisons that are perfectly sound: the first live A/B was blocked by six # infra_seed rows that every statistic already excludes. - surface_rows = [(line, record) for line, record in rows if not is_infra_error_row(record)] + surface_rows = [(line, record) for line, record in rows if not is_unlaunched_row(record)] manifest_values = _record_values(surface_rows, TOOL_MANIFEST_FIELD) if len(manifest_values) > 1: issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(manifest_values)}") diff --git a/evals/report/load.py b/evals/report/load.py index 4cab84f1..52c06c2c 100644 --- a/evals/report/load.py +++ b/evals/report/load.py @@ -177,6 +177,17 @@ def is_infra_error_row(row: ResultRow) -> bool: return isinstance(error_class, str) and error_class.startswith("infra_") +def is_unlaunched_row(row: ResultRow) -> bool: + """True when no agent ran for this row, so it observed no tool manifest. + + A seed failure and a seed-time skip both end before the agent starts. Neither can + carry a manifest fingerprint, so neither can be held to one — demanding it refused + perfectly sound comparisons: one plan-gated task made every report command exit. + """ + result = read_result(row) + return is_infra_error_row(row) or bool(result.skipped) + + def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: """Keep only the last row per (task_id, rep, label); preserve key insertion order.""" latest: dict[tuple[str, int, str], TaskResult] = {} diff --git a/evals/results.py b/evals/results.py index ab433ba0..1d4fe44d 100644 --- a/evals/results.py +++ b/evals/results.py @@ -631,8 +631,6 @@ def agent_run_to_task_result( cum_input = 0 cum_reason = None usage_per_iteration = [] - if run.usage and run.usage_scope == "iteration": - pass estimated_states = [bool(c.result_tokens_estimated) for c in calls] if estimated_states: diff --git a/evals/seed/plan.py b/evals/seed/plan.py index fc23018b..ce9b2834 100644 --- a/evals/seed/plan.py +++ b/evals/seed/plan.py @@ -58,6 +58,8 @@ def seed_plan(needs: set[str]) -> list[str]: "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " "(agent enables; teardown re-enables customers=True for later C1)" ) + elif "leave_worklogs_off" in needs: + lines.append("feature_exclusions (W11): project worklogs OFF (agent enables); workspace customers=True") else: lines.append( "workspace_features: customers=True " diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 3ed5fa81..d5c2b4a3 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -1580,3 +1580,36 @@ def test_codex_isolated_home_routes_approvals_through_automatic_review(tmp_path: ) config = (codex_home / "config.toml").read_text(encoding="utf-8") assert 'approvals_reviewer = "auto_review"' in config + + +def test_codex_nonzero_exit_is_not_scored_as_a_finished_attempt(tmp_path: Path): + """A failed codex process is infrastructure, not a model failure. + + Codex defaulted to ``end_turn`` whatever its exit code, so an authentication or network + failure that still emitted a partial ``agent_message`` was verified and counted in the + success rate. opencode and antigravity already map a nonzero exit to ``error``, so this + also skewed every driver comparison against them. + """ + driver = CodexCliDriver(codex_bin="/usr/bin/true", runner=lambda *_a, **_k: None, allow_live=True) + stdout = json.dumps({"type": "agent_message", "message": "partial"}) + "\n" + notes: list[str] = [] + launch = None + + completed = driver.parse_output( + subprocess.CompletedProcess(["codex"], 1, stdout=stdout, stderr="auth failed"), + launch=launch, + task_cwd=tmp_path, + max_turns=10, + notes=notes, + ) + assert completed.stopped_reason == "error" + assert any(note == "codex_exit=1" for note in notes) + + clean = driver.parse_output( + subprocess.CompletedProcess(["codex"], 0, stdout=stdout, stderr=""), + launch=launch, + task_cwd=tmp_path, + max_turns=10, + notes=[], + ) + assert clean.stopped_reason == "end_turn" diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py index c914d68e..26393f46 100644 --- a/tests/evals/report/test_identity.py +++ b/tests/evals/report/test_identity.py @@ -465,3 +465,27 @@ def test_infra_error_rows_without_a_manifest_do_not_refuse_the_comparison(tmp_pa ) report = identity.validate_persisted_identity([path]) assert report.files[0].values[identity.TOOL_MANIFEST_FIELD] == "fp-a" + + +def test_expected_skip_rows_without_a_manifest_do_not_refuse_the_comparison(tmp_path: Path): + """A plan-gated skip ends before the agent starts, so it carries no manifest either. + + Every report command exited 2 on any file mixing one such skip with evaluated rows, even + though summary semantics count an expected skip as a complete row. + """ + path = tmp_path / "rows.jsonl" + common = { + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + } + surface = {**common, "task_id": "R1", "tool_manifest_fingerprint": "fp-a", "success": True} + plan_gated = {**common, "task_id": "L4", "skipped": "env:plan-gated:customers"} + path.write_text(json.dumps(surface) + "\n" + json.dumps(plan_gated) + "\n", encoding="utf-8") + + report = identity.validate_persisted_identity([path]) + assert report.files[0].values[identity.TOOL_MANIFEST_FIELD] == "fp-a" diff --git a/tests/evals/test_cli.py b/tests/evals/test_cli.py index ee77b269..a27437f2 100644 --- a/tests/evals/test_cli.py +++ b/tests/evals/test_cli.py @@ -32,7 +32,8 @@ "C2", } -EXTRA_IDS = {"W9", "W10", "R7", "S5"} # bulk, pages, transitions, features +EXTRA_IDS = {"W9", "W10", "W11", "R7", "S5"} # bulk, pages, feature recovery, transitions, features +DEBIAS_IDS = {"I1", "I2", "I3", "I4", "I5", "L1", "L2", "L3", "L4", "L5"} @pytest.mark.parametrize("case", ["list-task-ids", "dry-run-all"]) @@ -40,7 +41,10 @@ def test_cmd_behaviours(case, capsys): if case == "list-task-ids": assert cmd_list() == 0 out = capsys.readouterr().out - for task_id in DESIGN_IDS | EXTRA_IDS: + # Every registered task, not a subset: the listing is how a caller discovers the + # battery, and a task missing from it is invisible. + assert DESIGN_IDS | EXTRA_IDS | DEBIAS_IDS == {task["id"] for task in TASKS} + for task_id in DESIGN_IDS | EXTRA_IDS | DEBIAS_IDS: assert task_id in out else: assert cmd_dry_run(list(TASKS)) == 0 From 12f95387c80b3eba2a117d889cf4128686dfc68e Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 14:49:08 +0530 Subject: [PATCH 57/93] Close the holes the regrade found in the new provenance rule Give L1 a worklog the agent did not create. Its answer was the id of the item it had just logged time on, and the write echoes that id, so both the answer and its provenance were satisfied without ever reading the project worklog summary the task exists to exercise. The seeded row is on an item the prompt never names. Count an undelivered line. Recording happens before forwarding so a fast child cannot race an unregistered pending id; the cost was that a broken pipe left a match in a sidecar still marked complete, proving surface use the agent never had. It now makes the sidecar non-authoritative. Restore the negative cases both transport tests lost: each had only responses that carry the sentinel, so a bug labelling every checked response would have passed, and the proxy had no wrong-target aggregate case at all. The retention claim was also wrong. A correct answer to R1 *is* the seeded state name, so sentinels do reach a row through final_text and verifier notes; only the response body and the evidence machinery are covered. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/DESIGN.md | 9 ++- evals/README.md | 4 +- evals/drivers/cli/sidecar.py | 3 + evals/proxy.py | 21 +++++- evals/runner/live.py | 2 +- evals/seed/work_items.py | 59 +++++++++++++++- evals/state_oracle.py | 24 ++++++- evals/tasks/answers.py | 7 +- evals/tasks/catalog.py | 7 +- evals/tasks/debias.py | 24 +++---- tests/evals/drivers/test_api_driver.py | 9 +-- tests/evals/seed/test_read_randomization.py | 22 +++++- tests/evals/tasks/test_catalog.py | 13 ++-- tests/evals/tasks/test_debias_verifiers.py | 18 ++--- .../evals/tasks/test_verifier_read_errors.py | 2 +- tests/evals/test_proxy.py | 68 ++++++++++++++++++- 16 files changed, 241 insertions(+), 51 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 039b5a11..e6851da0 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -156,9 +156,12 @@ Matching happens while the successful response is in memory, and only a non-sens back on. CLI proxies receive sentinel lengths and SHA-256 fingerprints, plus the target IDs the aggregate rule needs, through a mode-0600 run-scoped file outside the agent cwd; the raw value is absent even from that file. Every MCP proxy session reads the same file, and the driver -removes it with its temporary directory after the run. Neither the response body nor the -sentinel enters the result row. Unavailable or incomplete matching is diagnosed and cannot -pass a read verifier. +removes it with its temporary directory after the run. The response body never enters the +result row, and the evidence machinery persists only the matched label. The sentinel itself +is not secret after the fact: a correct answer to R1 *is* the seeded state name, so it +appears in the recorded `final_text`, and a failing verifier note names the value it wanted. +Result rows are therefore run data, not a redacted artifact. Unavailable or incomplete +matching is diagnosed and cannot pass a read verifier. ### Threat model: a cooperative agent diff --git a/evals/README.md b/evals/README.md index adf634c7..a5cf1ff9 100644 --- a/evals/README.md +++ b/evals/README.md @@ -130,7 +130,9 @@ both rules and the threat model they hold under. CLI proxies receive one-way value fingerprints, plus the target IDs the count rule needs, through a private run-scoped file; the raw sentinel is absent even if that file is inspected. Every proxy session can read it, and the driver removes it with its temporary directory after -the run. Result rows contain the matched label, never the sentinel value or response body. +the run. The evidence machinery records the matched label and never the response body. The +sentinel value can still reach a row by the front door: a correct answer often *is* the seeded +value, so it appears in `final_text`, and a failing verifier note names what it expected. Unavailable or incomplete matching is diagnosed and fails closed. ### Reading results diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 15e7f650..00632f51 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -190,6 +190,7 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] "non_json_lines", "malformed_jsonrpc", "recorder_errors", + "undelivered_lines", ) fatal_counts = ( "pending_left", @@ -197,6 +198,7 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] "unmatched_responses", "unparsed_lines", "recorder_errors", + "undelivered_lines", "invalid_seq", "duplicate_seq", "missing_seq", @@ -355,6 +357,7 @@ def _incompleteness_note(status: dict[str, Any]) -> str: "non_json_lines", "malformed_jsonrpc", "recorder_errors", + "undelivered_lines", "invalid_seq", "duplicate_seq", "missing_seq", diff --git a/evals/proxy.py b/evals/proxy.py index 4d976b48..4cf4a6d2 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -175,6 +175,7 @@ def __init__( self.non_json_lines = 0 self.malformed_jsonrpc = 0 self.recorder_errors = 0 + self.undelivered_lines = 0 self.unmatched_responses = 0 self.non_tool_responses = 0 self.notifications = 0 @@ -217,6 +218,17 @@ def note_recorder_error(self) -> None: with self._error_lock: self.recorder_errors += 1 + def note_undelivered(self) -> None: + """Count a line recorded here that never reached the other endpoint. + + Recording happens before forwarding so a fast child cannot race an unregistered + pending id. The cost is that a broken pipe leaves a match in the sidecar for a + response the agent never saw, which would prove surface use it never had. Counting + it makes the sidecar non-authoritative instead of quietly wrong. + """ + with self._error_lock: + self.undelivered_lines += 1 + def on_client_message(self, obj: dict[str, Any]) -> None: """Handle a parsed JSON-RPC message from the client (parent → child).""" has_method = "method" in obj @@ -335,6 +347,7 @@ def write_meta(self) -> None: return with self._error_lock: recorder_errors = self.recorder_errors + undelivered_lines = self.undelivered_lines row = { "row_type": "proxy_meta", "relayed_lines": self.relayed_lines, @@ -342,6 +355,7 @@ def write_meta(self) -> None: "non_json_lines": self.non_json_lines, "malformed_jsonrpc": self.malformed_jsonrpc, "recorder_errors": recorder_errors, + "undelivered_lines": undelivered_lines, "unmatched_responses": self.unmatched_responses, "non_tool_responses": self.non_tool_responses, "notifications": self.notifications, @@ -459,7 +473,12 @@ def process_buffer_lines( except Exception: recorder.note_recorder_error() # Forward only after recording so the opposite endpoint cannot race. - write_all_fd(forward_fd, line) + try: + write_all_fd(forward_fd, line) + except (BrokenPipeError, OSError): + if recorder is not None: + recorder.note_undelivered() + raise def pump_raw( diff --git a/evals/runner/live.py b/evals/runner/live.py index 998d74c2..e5e013bb 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -193,7 +193,7 @@ def _seed_fixtures( context.get("evidence_targets"), context.get("evidence_aggregates"), ): - raise RuntimeError(f"{task['id']} seed did not register target-bound response evidence") + raise RuntimeError(f"{task['id']} seed did not register any response evidence") except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 263b1e4e..bba7b1e2 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -9,7 +9,12 @@ from plane import PlaneClient from plane.models.query_params import WorkItemQueryParams from plane.models.states import CreateState -from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem +from plane.models.work_items import ( + CreateWorkItem, + CreateWorkItemComment, + CreateWorkItemWorkLog, + UpdateWorkItem, +) from evals.changelog import normalize_changelog_text from evals.errors import TaskSkipped @@ -27,8 +32,10 @@ UNFINISHED_CYCLE_TITLES, WORK_ITEM_FIXTURES, ) +from evals.state_oracle import worklog_summary_item_ids from .identities import record_seeded_entity +from .projects import plan_gate_skips from .randomize import random_truth_rng, random_truth_token, record_randomized_truth __all__ = [ @@ -365,8 +372,36 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, confirmed_id = str(getattr(detail, "id", None) or "") if confirmed_id != work_item_id: raise RuntimeError(f"seed L1: target work item readback id={confirmed_id!r}; want {work_item_id!r}") - context["l1_expected_summary_ids"] = [confirmed_id] - set_target_evidence(context, [confirmed_id]) + # Seed a worklog the agent did not create, on an item it is not told about. Without + # one, the summary contains only the row the agent just wrote, so reporting the id it + # already holds was both the correct answer and its own provenance — L1 could be + # passed without ever reading the summary it exists to exercise. + other_id = _second_worklog_item_id(context, exclude=confirmed_id) + seeded_minutes = rng.randrange(15, 240, 15) + with plan_gate_skips("worklogs"): + plane.work_items.work_logs.create( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=other_id, + data=CreateWorkItemWorkLog(duration=seeded_minutes, description=f"seeded {hidden_token}"), + ) + confirmed_summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) + confirmed_summary_ids = worklog_summary_item_ids(confirmed_summary) + if other_id not in confirmed_summary_ids: + raise RuntimeError( + f"seed L1: seeded worklog on {other_id} is absent from the project summary {confirmed_summary_ids!r}" + ) + record_randomized_truth( + context, + "L1.seeded_worklog", + {"intended_item": other_id, "intended_minutes": seeded_minutes}, + ) + context["randomized_truth"]["L1.seeded_worklog"]["confirmed"] = { + "summary_ids": list(confirmed_summary_ids), + } + # The agent's own 90-minute log adds the target row during the run. + context["l1_expected_summary_ids"] = sorted({*confirmed_summary_ids, confirmed_id}) + set_target_evidence(context, [other_id]) if task_id == "L5": attachment_target_id = context["fixture_item_ids"][PAYMENT_WEBHOOK_TITLE] @@ -407,6 +442,24 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, set_target_evidence(context, confirmed_names) +def _second_worklog_item_id(context: dict[str, Any], *, exclude: str) -> str: + """Pick the seeded item that carries L1's pre-existing worklog. + + Deterministic per run so the oracle is reproducible from the fixture seed, and never the + item the prompt names — the point is a summary row the agent can only learn by reading + the summary. + """ + candidates = sorted( + str(item_id) + for item_id in (context.get("fixture_item_ids") or {}).values() + if str(item_id) and str(item_id) != str(exclude) + ) + if not candidates: + raise RuntimeError("seed L1: no second work item available to carry a seeded worklog") + rng = random_truth_rng(context, "L1:second-worklog") + return rng.choice(candidates) + + def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: """Require L2's seeded comments to materialize as activities. diff --git a/evals/state_oracle.py b/evals/state_oracle.py index dc1fca03..92039070 100644 --- a/evals/state_oracle.py +++ b/evals/state_oracle.py @@ -1,4 +1,4 @@ -"""Neutral project-state normalization shared by seeders and verifiers.""" +"""Neutral Plane response normalization shared by seeders and verifiers.""" from __future__ import annotations @@ -18,4 +18,24 @@ def state_name_group_pairs(rows: list[Any]) -> list[str]: return pairs -__all__ = ["state_name_group_pairs"] +def worklog_summary_item_ids(summary: Any) -> list[str]: + """Return the distinct work item ids a project worklog summary reports, in order. + + Plane spells the field ``issue_id`` on some payload shapes and ``work_item_id`` on + others. The seeder builds L1's oracle from this and the verifier compares against it, + so they must read the payload identically. + """ + raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) + item_ids: list[str] = [] + for row in list(raw or []): + dump = row.model_dump() if hasattr(row, "model_dump") else (row if isinstance(row, dict) else {}) + value = getattr(row, "issue_id", None) or getattr(row, "work_item_id", None) + if value is None and isinstance(dump, dict): + value = dump.get("issue_id") or dump.get("work_item_id") + item_id = str(value or "").strip() + if item_id and item_id not in item_ids: + item_ids.append(item_id) + return item_ids + + +__all__ = ["state_name_group_pairs", "worklog_summary_item_ids"] diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py index 81f57797..00dd50ed 100644 --- a/evals/tasks/answers.py +++ b/evals/tasks/answers.py @@ -150,8 +150,9 @@ def answer_with_provenance( ) -> tuple[bool, str]: """Combine answer correctness with route-agnostic response evidence. - The two facts stay separate in the note. A successful unrelated call has no target - label and therefore cannot satisfy provenance. + The two facts stay separate in the note. Provenance needs a seeded value to have + appeared in a response the agent received; a run of successful calls that never + surfaced one does not satisfy it. """ calls = run.get("calls") source = str(run.get("call_source") or "unknown") @@ -161,7 +162,7 @@ def answer_with_provenance( if trace_incomplete: provenance_note = f"trace incomplete (source={source}; proxy sidecar was not authoritative)" elif provenance: - provenance_note = f"observed target-entity response evidence (source={source})" + provenance_note = f"observed seeded-value response evidence (source={source})" elif not available: provenance_note = f"unavailable (source={source}; response-evidence matching was not active)" elif isinstance(calls, list) and calls: diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index dc306efc..30818df4 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,9 +80,14 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 10 +CATALOG_REVISION = 11 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. +Revision 11 gives L1 a worklog it did not create, on an item it is not told about. Its answer +was the id of the item it had just logged time on, and that id is echoed by the write itself, +so both the answer and its provenance were satisfied without ever reading the project worklog +summary the task exists to exercise. L1 results are not comparable across this transition. + Revision 10 replaces the per-task list of accepted provenance shapes with one rule per kind of evidence. A sentinel is a per-run random string that exists only inside Plane, so its presence in a response the agent received proves surface use on its own; the request no longer has to diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 1f84fe98..e0bd17b4 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -17,6 +17,7 @@ W3_TITLE, W8_TITLE, ) +from evals.state_oracle import worklog_summary_item_ids from evals.tasks.answers import ( answer_with_provenance, contract_values, @@ -237,10 +238,17 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup if not wid: return answer_with_provenance(False, f"seed item {L1_TITLE!r} missing", run) expected_summary_ids = [str(value) for value in (ctx.get("l1_expected_summary_ids") or [])] - if expected_summary_ids != [str(wid)]: + if str(wid) not in expected_summary_ids: return answer_with_provenance( False, - f"L1 fixture oracle mismatch: summary ids={expected_summary_ids!r}; target={wid!r}", + f"L1 fixture oracle mismatch: summary ids={expected_summary_ids!r} omit target={wid!r}", + run, + ) + if len(expected_summary_ids) < 2: + return answer_with_provenance( + False, + f"L1 fixture error: the summary oracle {expected_summary_ids!r} holds only the agent's own " + "row, so the answer does not require reading the summary", run, ) # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). @@ -252,20 +260,10 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup try: summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) - raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) - sum_rows = list(raw or []) except Exception as exc: raise_verifier_read_error("L1", "reading the project worklog summary", exc) - summary_ids: list[str] = [] - for row in sum_rows: - dump = row.model_dump() if hasattr(row, "model_dump") else (row if isinstance(row, dict) else {}) - value = getattr(row, "issue_id", None) or getattr(row, "work_item_id", None) - if value is None and isinstance(dump, dict): - value = dump.get("issue_id") or dump.get("work_item_id") - item_id = str(value or "").strip() - if item_id and item_id not in summary_ids: - summary_ids.append(item_id) + summary_ids = worklog_summary_item_ids(summary) if str(wid) not in summary_ids: return answer_with_provenance( False, diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 679fe523..0f45ae63 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -467,7 +467,7 @@ def _api_driver_records_only_matching_evidence_labels(): session = FakeMcpSession( [ ToolResult(call_id="a", text=f"state={sentinel}"), - ToolResult(call_id="b", text=f"state={sentinel}"), + ToolResult(call_id="b", text="no seeded value in this response"), ] ) @@ -476,11 +476,12 @@ def _api_driver_records_only_matching_evidence_labels(): evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, ) - # The sentinel only exists inside Plane, so either response having it is proof of - # surface use; which entity the request named does not change that. + # The sentinel only exists inside Plane, so the response carrying it is proof of + # surface use even though its request named an unrelated entity. The response without + # it is not evidence, whatever it was asked about. assert run.evidence_trace_available is True assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] - assert run.calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.calls[1]["observed_sentinels"] == [] assert "result_text" not in run.calls[1] diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index 1093030d..ebdb9e45 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -46,6 +46,9 @@ def __init__(self): activities=SimpleNamespace(list=self._list_activities), attachments=SimpleNamespace(upload_from_bytes=self._upload_attachment, list=self._list_attachments), ) + self._work_logs: dict[str, list[SimpleNamespace]] = {} + self.work_items.work_logs = SimpleNamespace(create=self._create_work_log) + self.projects = SimpleNamespace(get_worklog_summary=self._worklog_summary) self.cycles = SimpleNamespace( create=self._create_cycle, update=self._update_cycle, @@ -54,6 +57,14 @@ def __init__(self): list_work_items=self._list_cycle_items, ) + def _create_work_log(self, *, work_item_id, data, **kwargs): + log = SimpleNamespace(id=f"log-{work_item_id}", duration=data.duration, description=data.description) + self._work_logs.setdefault(str(work_item_id), []).append(log) + return log + + def _worklog_summary(self, **kwargs): + return [SimpleNamespace(issue_id=item_id) for item_id in sorted(self._work_logs)] + def _create_state(self, *, data, **kwargs): state = SimpleNamespace( id=f"state-{len(self._states) + 1}", @@ -249,5 +260,12 @@ def test_l1_seed_oracle_is_the_api_confirmed_target_id(): seed_work_items(plane, "ws", ctx) - assert ctx["l1_expected_summary_ids"] == [ctx["fixture_item_ids"]["Payment webhook drops retries"]] - assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == tuple(ctx["l1_expected_summary_ids"]) + target_id = ctx["fixture_item_ids"]["Payment webhook drops retries"] + # The oracle holds the agent's own row plus a row it was never told about, so reporting + # the id it already has is no longer the whole answer. + assert target_id in ctx["l1_expected_summary_ids"] + assert len(ctx["l1_expected_summary_ids"]) == 2 + seeded_id = next(i for i in ctx["l1_expected_summary_ids"] if i != target_id) + assert plane._work_logs[seeded_id] + # Provenance is the seeded row's id: the target's id is echoed by the agent's own write. + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == (seeded_id,) diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index 37bd302d..ae683f95 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -98,9 +98,10 @@ # "7b8dc6bd2f8f" at revision 5 before unverifiable W8/W9 asks were removed. # "77230f96962d" at revision 6 before target-entity response evidence. # "ea5bdba36109" at revision 9 before R6 accepted per-project counts as provenance. +# "05176e65e594" at revision 10 before L1 got a worklog the agent did not create. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "05176e65e594" +PINNED_SYNTHETIC_BATTERY = "d9d087b763c8" @pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) @@ -318,11 +319,13 @@ def test_fingerprint_records_the_revision_transition(): full-catalog value was ``61bddef0dc76``. Revision 9 also binds R1/I2 provenance to the seeded state, because a work item's state is an id and resolving its name takes a second call; its full-catalog value was ``e9134604a0a7``. Revision 10 accepts one count per project as - R6 provenance, alongside the single count grouped by project it already accepted. Results - across these transitions are not comparable. + R6 provenance, alongside the single count grouped by project it already accepted; its + full-catalog value was ``9cce7ce77310``. Revision 11 gives L1 a worklog the agent did not + create, because reporting the id it had just logged time on required no summary read. + Results across these transitions are not comparable. Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 10 - assert battery_fingerprint() == "9cce7ce77310" + assert CATALOG_REVISION == 11 + assert battery_fingerprint() == "fa1784b052cd" diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py index d64e3c5e..6d644ddd 100644 --- a/tests/evals/tasks/test_debias_verifiers.py +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -263,27 +263,27 @@ async def _go(): "workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "wi-l1"}, - "l1_expected_summary_ids": ["wi-l1"], + "l1_expected_summary_ids": ["wi-l1", "wi-seeded"], } cases = [ ("untouched: no worklog", [], None, "", False, ()), - ("wrong duration 120", [120], ["wi-l1"], "Logged 120 minutes; summary ok.", False, ("90",)), - ("90m logged but empty answer", [90], ["wi-l1"], "", False, ("logged-minutes",)), + ("wrong duration 120", [120], ["wi-l1", "wi-seeded"], "Logged 120 minutes; summary ok.", False, ("90",)), + ("90m logged but empty answer", [90], ["wi-l1", "wi-seeded"], "", False, ("logged-minutes",)), ( "English 'ninety' is not a number", [90], - ["wi-l1"], + ["wi-l1", "wi-seeded"], "Logged one hundred ninety minutes. Project summary looks fine.", False, (), ), - ("correct facts, no contract", [90], ["wi-l1"], "Logged 1.5 hours total.", False, ()), - ("bare prose, no contract", [90], ["wi-l1"], "90 minutes of work", False, ("logged-minutes",)), + ("correct facts, no contract", [90], ["wi-l1", "wi-seeded"], "Logged 1.5 hours total.", False, ()), + ("bare prose, no contract", [90], ["wi-l1", "wi-seeded"], "90 minutes of work", False, ("logged-minutes",)), ( "exact contract", [90], - ["wi-l1"], - "logged-minutes: 90\nsummary-work-item-id: wi-l1", + ["wi-l1", "wi-seeded"], + "logged-minutes: 90\nsummary-work-item-id: wi-l1\nsummary-work-item-id: wi-seeded", True, (), ), @@ -296,7 +296,7 @@ async def _go(): assert s in note.lower(), f"{label}: {note}" # The 'ninety' case must name the duration it objected to. - plane = _L1Plane([90], summary_ids=["wi-l1"]) + plane = _L1Plane([90], summary_ids=["wi-l1", "wi-seeded"]) _, note = await verify_l1( plane, dict(ctx), _run("Logged one hundred ninety minutes. Project summary looks fine.") ) diff --git a/tests/evals/tasks/test_verifier_read_errors.py b/tests/evals/tasks/test_verifier_read_errors.py index 128486c9..e151375e 100644 --- a/tests/evals/tasks/test_verifier_read_errors.py +++ b/tests/evals/tasks/test_verifier_read_errors.py @@ -232,7 +232,7 @@ def add(case_id: str, task: str, reading: str, verifier: Any, plane: Any, ctx: d "workspace_slug": "ws", "project_id": "p1", "items": {L1_TITLE: "item-1"}, - "l1_expected_summary_ids": ["item-1"], + "l1_expected_summary_ids": ["item-1", "item-seeded"], }, ) add( diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index d8f4d30b..9c056a5f 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -741,11 +741,26 @@ def _sidecar_recorder_unit(tmp_path): "result": {"content": [{"type": "text", "text": f"target={sentinel}"}], "isError": False}, } ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 10, + "method": "tools/call", + "params": {"name": "t", "arguments": {"work_item_id": "target-1"}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 10, + "result": {"content": [{"type": "text", "text": "no seeded value here"}], "isError": False}, + } + ) rec.write_meta() calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] raw_calls = [row for row in raw_rows if row.get("row_type") != "proxy_meta"] - assert len(calls) == 2 + assert len(calls) == 3 assert calls[0]["tool"] == "t" assert calls[0]["args"] == {"work_item_id": "non-target"} assert calls[1]["args"] == {"work_item_id": "target-1"} @@ -754,11 +769,14 @@ def _sidecar_recorder_unit(tmp_path): assert all("result_text" not in row for row in raw_calls) # A sentinel is a per-run random string that exists only inside Plane, so its # presence proves the response came from the surface whichever entity the request - # named. Both calls received it; both are evidence. + # named. Call 0 named an unrelated entity and is still evidence; call 2 named the + # seeded one but never received the value, and is not. assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert calls[2]["observed_sentinels"] == [] assert raw_calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] assert raw_calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert raw_calls[2]["observed_sentinels"] == [] assert sentinel not in (tmp_path / "a.jsonl").read_text(encoding="utf-8") assert rec.finalized is True @@ -790,11 +808,29 @@ def test_proxy_records_exact_target_bound_aggregate_evidence_without_payload(tmp "result": {"content": [{"type": "text", "text": '{"total_count": 4}'}]}, } ) + # A count is guessable, so it is only evidence from a request naming a seeded entity. + # Without this case, dropping the target check from proxy wiring left every CLI test green. + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "count_work_items", "arguments": {"pql": 'project = "project-other"'}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 2, + "result": {"content": [{"type": "text", "text": '{"total_count": 4}'}]}, + } + ) rec.write_meta() calls = load_proxy_sidecar_calls(path) assert calls[0]["observed_sentinels"] == [] assert calls[0]["observed_aggregates"] == [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 4}] + assert calls[1]["observed_aggregates"] == [] persisted = path.read_text(encoding="utf-8") assert "result_text" not in persisted assert '"content"' not in persisted @@ -1897,3 +1933,31 @@ def test_cli_fallback_does_not_restore_trace_integrity(tmp_path: Path): assert applied.trace_integrity_reason == "recorder_loss" assert applied.tool_manifest_fingerprint is None assert "proxy_sidecar_deferred_to_cli_trace" in notes + + +def test_a_response_the_agent_never_received_is_not_authoritative_evidence(tmp_path: Path): + """Recording happens before forwarding, so a broken pipe can match what nobody saw. + + The ordering is deliberate — a fast child must not race an unregistered pending id — but + it meant a sentinel or count from a response that failed to reach the agent stayed in a + sidecar still marked complete, proving surface use the agent never had. + """ + recorder = SidecarRecorder(tmp_path / "undelivered.jsonl") + read_fd, write_fd = os.pipe() + os.close(read_fd) # nothing is listening: the forward will fail + line = json.dumps({"jsonrpc": "2.0", "id": 1, "result": {"content": []}}).encode() + b"\n" + + with pytest.raises((BrokenPipeError, OSError)): + process_buffer_lines( + bytearray(line), + forward_fd=write_fd, + recorder=recorder, + is_client=False, + record_jsonrpc=True, + ) + os.close(write_fd) + recorder.write_meta() + + _calls, status = load_proxy_sidecar(tmp_path / "undelivered.jsonl") + assert status["undelivered_lines"] == 1 + assert status["state"] == "incomplete" From 0e0a355ed62a8890db6afd4a52d0a5a1aab86f55 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 14:54:45 +0530 Subject: [PATCH 58/93] Skip the opencode config readback instead of failing on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test launches the real CLI in the scrubbed environment it is testing, and opencode can block on setup it cannot complete there. That made `pytest tests/evals` deterministically red on any machine with opencode installed. An unavailable CLI cannot answer the question, so it skips — never passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- tests/evals/drivers/test_cli_driver.py | 27 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index d5c2b4a3..592b7cfa 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -986,15 +986,24 @@ def test_opencode_isolated_environment_effective_mcp_server_list_is_exactly_plan encoding="utf-8", ) - observed = subprocess.run( - [opencode_bin, "debug", "config"], - cwd=launch.cwd, - env=launch.env, - text=True, - capture_output=True, - check=True, - timeout=15, - ) + # The isolated environment is the point of the test, and it is also why the readback can + # fail: opencode in a scrubbed HOME may block on setup it cannot complete. An unavailable + # CLI cannot answer the question, so skip — never pass — rather than leaving `pytest + # tests/evals` deterministically red on any machine where opencode is installed. + try: + observed = subprocess.run( + [opencode_bin, "debug", "config"], + cwd=launch.cwd, + env=launch.env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + except subprocess.TimeoutExpired: + pytest.skip("opencode did not answer 'debug config' in an isolated environment") + except subprocess.CalledProcessError as exc: + pytest.skip(f"opencode 'debug config' failed in an isolated environment: {exc.stderr or exc}") effective_config = json.loads(observed.stdout) assert sorted((effective_config.get("mcp") or {}).keys()) == ["plane"] From a508129b2ea83b2fe3c61674b8e0e633a0018fa8 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 15:51:18 +0530 Subject: [PATCH 59/93] Fix two fixtures the smoke run caught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit work_logs.create takes a plain mapping where its sibling resources take request models, so L1 seeded nothing and failed as infra on every repetition. The offline fake now asserts the argument is a mapping, so passing a model fails a test rather than a battery. R6's second project had no Bug type of its own. Work item types are project-owned unless the workspace owns them, and its bugs were created carrying the main project's type id, so an agent resolving 'Bug' inside that project counted zero and named the main one — always in that direction, which is what gave it away. The oracle read those ids back directly and disagreed, so a correct agent was scored wrong roughly half the time. L1's contract was ambiguous once its summary held more than one row: "exactly one 'logged-minutes: 90' line and one 'summary-work-item-id' line for every row" was read by every repetition as one logged-minutes line per row. The clauses are now separate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/seed/projects.py | 42 ++++++++++++++++++--- evals/seed/work_items.py | 10 ++--- evals/tasks/catalog.py | 15 +++++++- evals/tasks/debias.py | 8 ++-- tests/evals/seed/test_read_randomization.py | 4 +- tests/evals/seed/test_seed.py | 14 +++++++ tests/evals/tasks/test_catalog.py | 13 ++++--- 7 files changed, 82 insertions(+), 24 deletions(-) diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 0394d960..60e84993 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -219,6 +219,29 @@ def enable_project_features( ) +def _project_bug_type_id(plane: PlaneClient, workspace_slug: str, project_id: str) -> str: + """Resolve or create a project-owned Bug type, so each project answers for its own.""" + from plane.models.work_item_types import CreateWorkItemType + + from .item_types import BUG_TYPE_NAME, is_work_item_type_named + + existing = next( + ( + row + for row in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) + if is_work_item_type_named(row, BUG_TYPE_NAME) + ), + None, + ) + if existing is None: + existing = plane.work_item_types.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItemType(name=BUG_TYPE_NAME), + ) + return str(existing.id) + + def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: """Seed two API-confirmed Bug counts, randomising R6's winner per row.""" from .item_types import seed_item_type @@ -246,7 +269,12 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s if not bug_id: raise RuntimeError("seed second_project: bug_type required for R6 bug counts") - # Import workspace-level type into second project when needed. + # Give the second project a Bug type of its own. Workspace-owned types are shared, so + # importing is enough; project-owned types are not, and creating B's items with the main + # project's type id left them invisible to an agent that resolves 'Bug' inside B. It + # counted zero there and named the main project, always in that direction, while the + # oracle — which reads those ids back directly — saw the seeded count and disagreed. + second_bug_id = bug_id if context.get("bug_type_workspace_level"): try: plane.work_item_types.import_to_project( @@ -259,6 +287,8 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s # May already be imported. if not (isinstance(exc, HttpError) and exc.status_code in (400, 409)): raise + else: + second_bug_id = _project_bug_type_id(plane, workspace_slug, str(project.id)) main_id = context["project_id"] task_id = str(context.get("task_id") or "") @@ -293,7 +323,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s item = plane.work_items.create( workspace_slug=workspace_slug, project_id=project.id, - data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + data=CreateWorkItem(name=title, priority="high", type_id=str(second_bug_id)), # type: ignore[arg-type] ) second_bug_ids.append(item.id) record_seeded_entity(context, "work_item", item.id) @@ -303,7 +333,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s context["r6_more_bugs_project"] = name return - def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: + def confirmed_open_bug_count(project_id: str, work_item_ids: list[str], type_id: str) -> int: count = 0 for work_item_id in work_item_ids: detail = plane.work_items.retrieve( @@ -311,15 +341,15 @@ def confirmed_open_bug_count(project_id: str, work_item_ids: list[str]) -> int: project_id=project_id, work_item_id=work_item_id, ) - if str(getattr(detail, "type_id", None) or "") != str(bug_id): + if str(getattr(detail, "type_id", None) or "") != str(type_id): continue if getattr(detail, "completed_at", None) or getattr(detail, "archived_at", None): continue count += 1 return count - confirmed_main = confirmed_open_bug_count(main_id, main_bug_ids) - confirmed_second = confirmed_open_bug_count(str(project.id), second_bug_ids) + confirmed_main = confirmed_open_bug_count(main_id, main_bug_ids, str(bug_id)) + confirmed_second = confirmed_open_bug_count(str(project.id), second_bug_ids, str(second_bug_id)) if confirmed_main == confirmed_second: raise RuntimeError(f"seed R6: API-confirmed open Bug counts tie ({confirmed_main} each)") main_project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=main_id) diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index bba7b1e2..21d818aa 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -9,12 +9,7 @@ from plane import PlaneClient from plane.models.query_params import WorkItemQueryParams from plane.models.states import CreateState -from plane.models.work_items import ( - CreateWorkItem, - CreateWorkItemComment, - CreateWorkItemWorkLog, - UpdateWorkItem, -) +from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem from evals.changelog import normalize_changelog_text from evals.errors import TaskSkipped @@ -383,7 +378,8 @@ def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, workspace_slug=workspace_slug, project_id=project_id, work_item_id=other_id, - data=CreateWorkItemWorkLog(duration=seeded_minutes, description=f"seeded {hidden_token}"), + # This resource takes a plain mapping, not a request model like its siblings. + data={"duration": seeded_minutes, "description": f"seeded {hidden_token}"}, ) confirmed_summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) confirmed_summary_ids = worklog_summary_item_ids(confirmed_summary) diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py index 30818df4..2cb28171 100644 --- a/evals/tasks/catalog.py +++ b/evals/tasks/catalog.py @@ -80,9 +80,22 @@ def task_author(task: dict[str, Any]) -> str: return str(task.get("author") or "claude") -CATALOG_REVISION = 11 +CATALOG_REVISION = 12 """Bumped when a deliberate change to a fixture or verifier redefines what a task asks. +Revision 12 also disambiguates L1's answer contract. It asked for "exactly one +'logged-minutes: 90' line and one 'summary-work-item-id' line for every row", which was +unambiguous only while the summary held a single row. With a seeded second row every agent read +"for every row" as governing both clauses and emitted one logged-minutes line per row — wrong for +a row whose seeded duration is not 90. The prompt now separates the two clauses and says other +items may already carry worklogs. + +Revision 12 gives R6's second project a Bug type of its own. Work item types are project-owned +unless the workspace owns them, so creating that project's bugs with the main project's type id +left them invisible to an agent resolving 'Bug' inside it: the agent counted zero there and named +the main project, always in that direction, while the oracle read those ids back directly and +disagreed. The seeded counts are unchanged; what changes is that the answer is now findable. + Revision 11 gives L1 a worklog it did not create, on an item it is not told about. Its answer was the id of the item it had just logged time on, and that id is echoed by the write itself, so both the answer and its provenance were satisfied without ever reading the project worklog diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index e0bd17b4..93d3ff1b 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -305,10 +305,10 @@ async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tup "tags": {"write", "read", "long_tail", "debias"}, "prompt": ( f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " - f"'{L1_TITLE}', then report the project's worklog summary. End with exactly " - "one 'logged-minutes: 90' line and one " - "'summary-work-item-id: ' line for every row returned " - "by the project worklog summary. Include no other lines with those prefixes." + f"'{L1_TITLE}', then report the project's worklog summary. End with exactly one " + "'logged-minutes: 90' line, and then, for each row the project worklog summary " + "returns, one 'summary-work-item-id: ' line. The project " + "may already have worklogs on other items. Include no other lines with those prefixes." ), "needs": {"items"}, "verify": verify_l1, diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index ebdb9e45..97d427ec 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -58,7 +58,9 @@ def __init__(self): ) def _create_work_log(self, *, work_item_id, data, **kwargs): - log = SimpleNamespace(id=f"log-{work_item_id}", duration=data.duration, description=data.description) + # The SDK resource takes a mapping here, so the fake must reject a model. + assert isinstance(data, dict), f"work_logs.create needs a mapping, got {type(data).__name__}" + log = SimpleNamespace(id=f"log-{work_item_id}", duration=data["duration"], description=data["description"]) self._work_logs.setdefault(str(work_item_id), []).append(log) return log diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index 48530034..e5932563 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -123,9 +123,23 @@ def seeded(run_id: str, *, mark_second_non_bug: int = 0): main_id = f"main-{run8}" main_name = f"EVAL {run8}" work_items = WorkItems(mark_second_non_bug=mark_second_non_bug) + # Types are project-owned in this fake, so the second project must be given its own + # Bug type: creating its items with the main project's type id is what made an agent + # resolving 'Bug' inside that project count zero. + project_types: dict[str, str] = {} + + def create_project_type(*, project_id, data, **kwargs): + type_id = f"bug-{project_id}" + project_types[str(project_id)] = type_id + return SimpleNamespace(id=type_id, name=data.name) + plane = SimpleNamespace( projects=Projects(main_id, main_name), work_items=work_items, + work_item_types=SimpleNamespace( + list=lambda **kwargs: [], + create=create_project_type, + ), ) ctx = { "run_id": run_id, diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py index ae683f95..9068bbf4 100644 --- a/tests/evals/tasks/test_catalog.py +++ b/tests/evals/tasks/test_catalog.py @@ -99,9 +99,10 @@ # "77230f96962d" at revision 6 before target-entity response evidence. # "ea5bdba36109" at revision 9 before R6 accepted per-project counts as provenance. # "05176e65e594" at revision 10 before L1 got a worklog the agent did not create. +# "d9d087b763c8" at revision 11 before R6's second project got a Bug type of its own. # This pin moves with every deliberate revision bump, and must not move otherwise — an # unexplained change means the serialization drifted, which is what the pin exists to catch. -PINNED_SYNTHETIC_BATTERY = "d9d087b763c8" +PINNED_SYNTHETIC_BATTERY = "7d77116e3299" @pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) @@ -321,11 +322,13 @@ def test_fingerprint_records_the_revision_transition(): its full-catalog value was ``e9134604a0a7``. Revision 10 accepts one count per project as R6 provenance, alongside the single count grouped by project it already accepted; its full-catalog value was ``9cce7ce77310``. Revision 11 gives L1 a worklog the agent did not - create, because reporting the id it had just logged time on required no summary read. - Results across these transitions are not comparable. + create, because reporting the id it had just logged time on required no summary read; its + full-catalog value was ``fa1784b052cd``. Revision 12 gives R6's second project a Bug type of + its own, because its bugs carried the main project's type and an agent resolving 'Bug' there + counted zero. Results across these transitions are not comparable. Asserting the constant rather than merely 'it changed' makes future drift visible. """ from evals.tasks.catalog import CATALOG_REVISION - assert CATALOG_REVISION == 11 - assert battery_fingerprint() == "fa1784b052cd" + assert CATALOG_REVISION == 12 + assert battery_fingerprint() == "eaf35e8019aa" From 2f65d15d633cf2f632a9413d621846ef92c0c3d3 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 15:56:34 +0530 Subject: [PATCH 60/93] Keep the refusal that caused a skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One L1 repetition skipped as plan-gated on worklogs while the next two seeded and passed in the same workspace, and the reason string was all that survived — so whether it was a plan limit, a feature toggle, or a transient failure is now unknowable. The status code and message travel beside the reason, which stays byte-stable because the skip taxonomy matches it exactly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/errors.py | 13 ++++++++++--- evals/runner/live.py | 12 ++++++++++-- evals/seed/projects.py | 4 +++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/evals/errors.py b/evals/errors.py index 54520998..21063180 100644 --- a/evals/errors.py +++ b/evals/errors.py @@ -4,11 +4,18 @@ class TaskSkipped(Exception): - """A task that cannot run in this environment without blaming the agent.""" + """A task that cannot run in this environment without blaming the agent. - def __init__(self, reason: str) -> None: + ``reason`` is matched exactly by the skip taxonomy and must stay stable, so the + refusal that caused the skip travels in ``detail`` instead. Without it, an + intermittent gate is unexplainable after the fact: the status code that would say + whether it was a plan limit, a feature toggle, or a transient failure is gone. + """ + + def __init__(self, reason: str, *, detail: str | None = None) -> None: self.reason = str(reason) - super().__init__(self.reason) + self.detail = str(detail) if detail else None + super().__init__(self.reason if not self.detail else f"{self.reason} ({self.detail})") __all__ = ["TaskSkipped"] diff --git a/evals/runner/live.py b/evals/runner/live.py index e5e013bb..92ef0687 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -197,7 +197,11 @@ def _seed_fixtures( except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}", flush=True) + print( + f" {task['id']} rep={repetition} SKIPPED: {skip.reason}" + + (f" — {skip.detail}" if getattr(skip, "detail", None) else ""), + flush=True, + ) return False except Exception as exc: row.success = False @@ -397,7 +401,11 @@ async def _verify_task( except TaskSkipped as skip: row.skipped = skip.reason row.verify_note = skip.reason - print(f" {task['id']} rep={repetition} SKIPPED: {skip.reason}", flush=True) + print( + f" {task['id']} rep={repetition} SKIPPED: {skip.reason}" + + (f" — {skip.detail}" if getattr(skip, "detail", None) else ""), + flush=True, + ) except Exception as exc: row.success = False row.error = f"{type(exc).__name__}: {exc}" diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 60e84993..5fc5327a 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -73,7 +73,9 @@ def plan_gate_skips(feature: str) -> Iterator[None]: yield except Exception as exc: if is_plan_gate(exc): - raise TaskSkipped(f"env:plan-gated:{feature}") from exc + status = getattr(exc, "status_code", None) + detail = f"HTTP {status}: {exc}" if status else str(exc) + raise TaskSkipped(f"env:plan-gated:{feature}", detail=detail[:300]) from exc raise From e34a877e020dc05a47a5c21f66ce862dd477f6ae Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 20:20:28 +0530 Subject: [PATCH 61/93] Give the plan-gate policy and the server env their own modules is_plan_gate lived in seed/projects, so every seeder that could meet a paid feature imported the project resource module to reach it. That made projects a policy hub and produced the package's only import cycle, since item_types needs the classifier while projects needs the item-type seeder. Both now import seed/gates. stdio_server_env lived in runner/live, the live-run composition root, so importing a pure token-counting helper loaded 71 evals modules. It is now a leaf: that import loads 3. Source no longer routes TaskSkipped through evals/tasks/skip.py. That path shipped and a compat test pins it, but everything imported it while the canonical evals/errors was imported by almost nothing, which made the shim look like the real module. The layering itself was already sound in every direction that matters, so the new boundary tests pin it rather than change it: offline reporting must not drag in live-run code, fixtures must not drag in agent backends, and the proxy must not drag in either. One test proves the probe can observe a violation, so a boundary test that cannot fail does not pass silently. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/listing.py | 2 +- evals/runner/canary.py | 2 +- evals/runner/live.py | 19 +------- evals/seed/__init__.py | 3 +- evals/seed/customers.py | 2 +- evals/seed/gates.py | 61 ++++++++++++++++++++++++++ evals/seed/item_types.py | 2 +- evals/seed/projects.py | 47 +------------------- evals/seed/releases.py | 2 +- evals/seed/work_items.py | 2 +- evals/server_env.py | 30 +++++++++++++ evals/tasks/__init__.py | 2 +- evals/tasks/schema.py | 2 +- evals/tasks/skip.py | 8 +++- tests/evals/test_package_boundaries.py | 52 ++++++++++++++++++++++ 15 files changed, 163 insertions(+), 73 deletions(-) create mode 100644 evals/seed/gates.py create mode 100644 evals/server_env.py create mode 100644 tests/evals/test_package_boundaries.py diff --git a/evals/listing.py b/evals/listing.py index d1a64ba9..2f50c600 100644 --- a/evals/listing.py +++ b/evals/listing.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from typing import Any -from evals.runner.live import stdio_server_env +from evals.server_env import stdio_server_env @dataclass diff --git a/evals/runner/canary.py b/evals/runner/canary.py index dd625bf6..44fb57cb 100644 --- a/evals/runner/canary.py +++ b/evals/runner/canary.py @@ -6,9 +6,9 @@ import uuid from typing import Any +from evals.errors import TaskSkipped from evals.seed import make_plane_client, seed, teardown from evals.tasks.catalog import battery_fingerprint -from evals.tasks.skip import TaskSkipped CANARY_CANNED_OUTPUTS: dict[str, tuple[str, ...]] = { "R1": ("state: In Progress",), diff --git a/evals/runner/live.py b/evals/runner/live.py index 92ef0687..38083ecc 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -4,7 +4,6 @@ import asyncio import json -import os import sys import time import uuid @@ -14,6 +13,7 @@ from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS +from evals.errors import TaskSkipped from evals.evidence import configured_evidence_labels from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys from evals.report.off_surface import off_surface_statement @@ -22,9 +22,9 @@ from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown from evals.seed.identities import capture_seed_artifacts +from evals.server_env import stdio_server_env from evals.tasks.catalog import battery_fingerprint, task_author, task_fingerprint from evals.tasks.prompts import PromptBindError, format_task_prompt -from evals.tasks.skip import TaskSkipped from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision from .resume import load_resume_skip_keys @@ -42,21 +42,6 @@ def _system_preamble(workspace_slug: str, project_name: str) -> str: ) -def stdio_server_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: - """Build MCP stdio env from scratch — never inherit os.environ (F6).""" - environment: dict[str, str] = {} - if path := os.environ.get("PATH"): - environment["PATH"] = path - if home := os.environ.get("HOME"): - environment["HOME"] = home - environment["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] - environment["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] - environment["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") - if extra: - environment.update(extra) - return environment - - def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: """True when a CLI AgentRun stop_reason should be classified as infra_cli. diff --git a/evals/seed/__init__.py b/evals/seed/__init__.py index 25307e34..a7c13acf 100644 --- a/evals/seed/__init__.py +++ b/evals/seed/__init__.py @@ -13,6 +13,7 @@ EVALUATION_CUSTOMER_PROPERTY_NAME as DEBIAS_CUSTOMER_PROP_DISPLAY, ) from .cycles import CYCLE_CURRENT, CYCLE_PAST, seed_cycles +from .gates import is_plan_gate, plan_gate_skips from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, seed_intake from .item_types import ( BUG_TYPE_NAME, @@ -31,7 +32,6 @@ enable_project_features, enable_workspace_features, is_identifier_collision, - is_plan_gate, secrets, seed_second_project, ) @@ -165,6 +165,7 @@ "is_identifier_collision", "is_evaluation_customer_name", "is_plan_gate", + "plan_gate_skips", "list_states", "make_plane_client", "require_activities", diff --git a/evals/seed/customers.py b/evals/seed/customers.py index a34200a9..35057215 100644 --- a/evals/seed/customers.py +++ b/evals/seed/customers.py @@ -14,8 +14,8 @@ is_evaluation_customer_name, ) +from .gates import plan_gate_skips from .identities import record_seeded_entity -from .projects import plan_gate_skips __all__ = [ "CUSTOMER_NAME", diff --git a/evals/seed/gates.py b/evals/seed/gates.py new file mode 100644 index 00000000..35c21d23 --- /dev/null +++ b/evals/seed/gates.py @@ -0,0 +1,61 @@ +"""Plan-gate classification, shared by every seeder that can meet a paid feature. + +This is policy, not a resource. It lived in ``projects`` because the first gate encountered +was a project one, and every other seeder then imported the project module to reach it — +which made ``projects`` a hub and produced the package's only import cycle, since +``item_types`` needs the classifier while ``projects`` needs the item-type seeder. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator + +from plane.errors.errors import HttpError + +from evals.errors import TaskSkipped + +# Wording a refusal uses when the workspace's plan is what stands in the way. A feature +# switched off for a project says "not enabled for this project" instead, which is a +# configuration state the harness can change and so is not a gate. +PLAN_GATE_PROSE = ("upgrade your plan", "payment required", "subscription", "not available on your") + + +def is_plan_gate(exc: BaseException) -> bool: + """True only for genuine plan gates — not generic API failures. + + 402 is unambiguous. 403 and 400 are not: Plane uses 403 for ordinary permission denial + and for the initiative/teamspace plan gates in the same shape, so a bare 403 counted as + a gate turned real permission bugs into environment skips. Those two now need the + refusal to name a plan limit. + """ + if not isinstance(exc, HttpError): + return False + if exc.status_code == 402: + return True + if exc.status_code not in (400, 403): + return False + blob = f"{exc} {exc.response!s}".lower() + return any(phrase in blob for phrase in PLAN_GATE_PROSE) + + +@contextlib.contextmanager +def plan_gate_skips(feature: str) -> Iterator[None]: + """Turn a plan refusal raised inside the block into a task skip. + + An uncaught seed exception becomes infra_seed and kills the task-rep; a capability the + plan excludes is an environment fact, recorded like L2's missing activity worker. + ``TaskSkipped`` lives in a neutral module, so seed and task packages can import in + either order without a cycle. + """ + try: + yield + except Exception as exc: + if is_plan_gate(exc): + status = getattr(exc, "status_code", None) + detail = f"HTTP {status}: {exc}" if status else str(exc) + raise TaskSkipped(f"env:plan-gated:{feature}", detail=detail[:300]) from exc + raise + + +__all__ = ["PLAN_GATE_PROSE", "is_plan_gate", "plan_gate_skips"] diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py index 1643e12a..3bab4719 100644 --- a/evals/seed/item_types.py +++ b/evals/seed/item_types.py @@ -7,8 +7,8 @@ from plane import PlaneClient from plane.models.work_item_types import CreateWorkItemType +from .gates import is_plan_gate from .identities import record_seeded_entity -from .projects import is_plan_gate BUG_TYPE_NAME = "Bug" INCIDENT_TYPE_NAME = "Incident" diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 5fc5327a..229df2a9 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -2,9 +2,7 @@ from __future__ import annotations -import contextlib import secrets -from collections.abc import Iterator from typing import Any from plane import PlaneClient @@ -13,9 +11,9 @@ from plane.models.work_items import CreateWorkItem from plane.models.workspaces import WorkspaceFeature -from evals.errors import TaskSkipped from evals.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence +from .gates import is_plan_gate from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth @@ -36,49 +34,6 @@ ) -# Wording a refusal uses when the workspace's plan is what stands in the way. A feature -# switched off for a project says "not enabled for this project" instead, which is a -# configuration state the harness can change and so is not a gate. -PLAN_GATE_PROSE = ("upgrade your plan", "payment required", "subscription", "not available on your") - - -def is_plan_gate(exc: BaseException) -> bool: - """True only for genuine plan gates — not generic API failures. - - 402 is unambiguous. 403 and 400 are not: Plane uses 403 for ordinary permission denial - and for the initiative/teamspace plan gates in the same shape, so a bare 403 counted as - a gate turned real permission bugs into environment skips. Those two now need the - refusal to name a plan limit. - """ - if not isinstance(exc, HttpError): - return False - if exc.status_code == 402: - return True - if exc.status_code not in (400, 403): - return False - blob = f"{exc} {exc.response!s}".lower() - return any(phrase in blob for phrase in PLAN_GATE_PROSE) - - -@contextlib.contextmanager -def plan_gate_skips(feature: str) -> Iterator[None]: - """Turn a plan refusal raised inside the block into a task skip. - - An uncaught seed exception becomes infra_seed and kills the task-rep; a capability the - plan excludes is an environment fact, recorded like L2's missing activity worker. - ``TaskSkipped`` lives in a neutral module, so seed and task packages can import in - either order without a cycle. - """ - try: - yield - except Exception as exc: - if is_plan_gate(exc): - status = getattr(exc, "status_code", None) - detail = f"HTTP {status}: {exc}" if status else str(exc) - raise TaskSkipped(f"env:plan-gated:{feature}", detail=detail[:300]) from exc - raise - - def is_identifier_collision(exc: BaseException) -> bool: """True when project create failed because the identifier is already taken. diff --git a/evals/seed/releases.py b/evals/seed/releases.py index 4a6039c4..138d9f90 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -15,8 +15,8 @@ RELEASE_NAME, ) +from .gates import plan_gate_skips from .identities import record_seeded_entity -from .projects import plan_gate_skips from .randomize import random_truth_rng, random_truth_token, record_randomized_truth __all__ = [ diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index 21d818aa..a616166d 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -29,8 +29,8 @@ ) from evals.state_oracle import worklog_summary_item_ids +from .gates import plan_gate_skips from .identities import record_seeded_entity -from .projects import plan_gate_skips from .randomize import random_truth_rng, random_truth_token, record_randomized_truth __all__ = [ diff --git a/evals/server_env.py b/evals/server_env.py new file mode 100644 index 00000000..65bc05a6 --- /dev/null +++ b/evals/server_env.py @@ -0,0 +1,30 @@ +"""Environment construction for a stdio MCP server child process. + +A foundational leaf, shared by the live runner and the standalone tool-token listing. It +lived in ``runner.live``, so importing a pure token-counting helper pulled in the whole +live-run composition root — every driver, seeder, task and report module with it. +""" + +from __future__ import annotations + +import os + +DEFAULT_PLANE_BASE_URL = "https://api.plane.so" + + +def stdio_server_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6).""" + environment: dict[str, str] = {} + if path := os.environ.get("PATH"): + environment["PATH"] = path + if home := os.environ.get("HOME"): + environment["HOME"] = home + environment["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] + environment["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + environment["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", DEFAULT_PLANE_BASE_URL) + if extra: + environment.update(extra) + return environment + + +__all__ = ["DEFAULT_PLANE_BASE_URL", "stdio_server_env"] diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index 9a1e6d96..ac57f436 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -1,5 +1,6 @@ """Public task catalog and verifier API.""" +from evals.errors import TaskSkipped from evals.tasks.answers import ( contract_values, get_final_text, @@ -57,7 +58,6 @@ from evals.tasks.prompts import PromptBindError, format_task_prompt from evals.tasks.read import verify_r1, verify_r2, verify_r3, verify_r4, verify_r5, verify_r6, verify_r7 from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 -from evals.tasks.skip import TaskSkipped from evals.tasks.write import ( verify_w1, verify_w2, diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py index ee2555d3..31d6bb54 100644 --- a/evals/tasks/schema.py +++ b/evals/tasks/schema.py @@ -7,9 +7,9 @@ from plane.errors.errors import HttpError from plane.models.enums import PropertyType +from evals.errors import TaskSkipped from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE from evals.tasks.lookups import as_id, find_item_by_name -from evals.tasks.skip import TaskSkipped from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error diff --git a/evals/tasks/skip.py b/evals/tasks/skip.py index 29e0a1d8..0822efb5 100644 --- a/evals/tasks/skip.py +++ b/evals/tasks/skip.py @@ -1,4 +1,10 @@ -"""Backward-compatible task-skip import path.""" +"""Retired import path for :class:`evals.errors.TaskSkipped`. + +Kept because it shipped and ``tests/evals/test_import_compat.py`` pins it. Nothing in the +package imports it: the canonical home is ``evals.errors``, which depends on nothing. Every +source module routed through here for a while, which left the neutral module unused and made +a compat shim look like the real one. +""" from evals.errors import TaskSkipped as TaskSkipped diff --git a/tests/evals/test_package_boundaries.py b/tests/evals/test_package_boundaries.py new file mode 100644 index 00000000..b8ec3621 --- /dev/null +++ b/tests/evals/test_package_boundaries.py @@ -0,0 +1,52 @@ +"""Import-closure tests for the package's layering. + +The direction of these edges is the part of the structure worth defending: offline reporting +must be usable without live-run code, fixtures without agent backends, and the recording +proxy without any of it. Every invariant here held when the tests were written, so a failure +means a new import changed the shape of the package rather than a pre-existing violation. + +Each case runs in a fresh interpreter, because import closure is a property of a process. +""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + +# (module to import, package names it must not drag in) +BOUNDARIES = [ + ("evals.report.load", ("runner", "drivers", "seed", "proxy")), + ("evals.report.statistics", ("runner", "drivers", "seed")), + ("evals.seed.build", ("drivers", "report", "runner")), + ("evals.proxy", ("runner", "drivers", "seed", "tasks", "report")), + ("evals.tasks", ("runner", "drivers", "report", "proxy")), + # A pure token-counting helper once imported the live runner, so importing it loaded + # every driver, seeder, task and report module in the tree. + ("evals.listing", ("runner", "drivers", "seed", "tasks", "report")), + # The neutral exception module is the floor: it may depend on nothing of ours. + ("evals.errors", ("runner", "drivers", "seed", "tasks", "report", "proxy", "results")), +] + + +def loaded_subpackages(module: str) -> set[str]: + """Return the ``evals.*`` subpackages present in sys.modules after importing ``module``.""" + code = ( + f"import {module}, sys\n" + "print(' '.join(sorted({m.split('.')[1] for m in sys.modules " + "if m.startswith('evals.') and m.count('.') >= 1})))" + ) + completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + return set(completed.stdout.split()) + + +@pytest.mark.parametrize(("module", "forbidden"), BOUNDARIES, ids=[case[0] for case in BOUNDARIES]) +def test_import_does_not_cross_layer(module: str, forbidden: tuple[str, ...]): + leaked = loaded_subpackages(module) & set(forbidden) + assert not leaked, f"importing {module} loaded {sorted(leaked)}, which it must not depend on" + + +def test_the_probe_can_actually_observe_a_violation(): + """A boundary test that cannot fail is worse than none: prove the probe sees imports.""" + assert "runner" in loaded_subpackages("evals.runner.live") From 55d98423b692fc01320a31015c5e4164fc734451 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 20:34:05 +0530 Subject: [PATCH 62/93] Report the run that was executed, not the checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reports read three task facts from the live catalog: whether a task mutates Plane, its prompt text, and the fixtures it needs, which decides whether a plan-gated skip was expected. All three are properties of the run, so reading them from the working tree meant an old result file could be reinterpreted after the catalog changed — the exact drift the battery fingerprint and identity validation exist to prevent. The meta header now carries them, and report/ no longer imports evals.tasks at all. skip_taxonomy keeps a fallback for files written before the header existed, which is the only remaining path to the catalog and is now reached only by them. A file with no metadata cannot support the write-without-write-call indicator, so that indicator says "not evaluated" instead of printing a zero. A silent zero reads as checked-and-clean, which is the opposite of unknown. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01J5kpUxiB9fd9apXJP6MA7K --- evals/report/off_surface.py | 29 ++++++-- evals/report/summary.py | 15 +++- evals/report/table.py | 23 ++++-- evals/runner/live.py | 2 + evals/runner/meta.py | 5 ++ evals/skip_taxonomy.py | 31 +++++--- evals/task_metadata.py | 97 ++++++++++++++++++++++++++ tests/evals/report/test_off_surface.py | 5 +- tests/evals/report/test_summary.py | 3 +- tests/evals/report/test_table.py | 11 ++- 10 files changed, 197 insertions(+), 24 deletions(-) create mode 100644 evals/task_metadata.py diff --git a/evals/report/off_surface.py b/evals/report/off_surface.py index 2fd16e42..99cc50c8 100644 --- a/evals/report/off_surface.py +++ b/evals/report/off_surface.py @@ -9,7 +9,7 @@ from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, CallRecord, TaskResult -from evals.tasks import TASKS_BY_ID +from evals.task_metadata import task_metadata_from_rows from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import percentile @@ -98,6 +98,8 @@ class OffSurfaceMeasurement: """Per-row findings and their run-level aggregate views.""" rows: tuple[OffSurfaceRow, ...] = () + mutation_intent_available: bool = True + """False when no run declared its task tags, so write-intent cannot be judged.""" @property def flagged_rows(self) -> int: @@ -125,7 +127,7 @@ def _successful_row(row: TaskResult) -> bool: def task_requires_mutation(task: Mapping[str, Any] | None) -> bool: - """Derive mutation intent from catalog tags instead of a task-id allowlist.""" + """Derive mutation intent from persisted tags instead of a task-id allowlist.""" tags = task.get("tags") if task is not None else () return bool(_MUTATING_TASK_TAGS.intersection(str(tag) for tag in (tags or ()))) @@ -157,9 +159,18 @@ def _answer_was_correct(row: TaskResult) -> bool: def measure_off_surface( rows: list[ResultRow], *, - task_catalog: Mapping[str, Mapping[str, Any]] = TASKS_BY_ID, + task_catalog: Mapping[str, Mapping[str, Any]] | None = None, ) -> OffSurfaceMeasurement: - """Compute suspicion indicators without changing row success or completeness.""" + """Compute suspicion indicators without changing row success or completeness. + + ``task_catalog`` carries the run's own task facts. When absent it falls back to the + metadata persisted in the rows' meta header; a file written before that header existed + yields no mutation intent, which suppresses one indicator rather than inventing it from + a catalog that may have changed since. + """ + if task_catalog is None: + task_catalog = task_metadata_from_rows(rows) + mutation_intent_available = bool(task_catalog) results: list[TaskResult] = [] for raw_row in rows: if is_meta_row(raw_row): @@ -211,7 +222,7 @@ def measure_off_surface( for index, row in enumerate(results) if flags_by_index[index] ) - return OffSurfaceMeasurement(rows=findings) + return OffSurfaceMeasurement(rows=findings, mutation_intent_available=mutation_intent_available) def off_surface_statement(measurement: OffSurfaceMeasurement) -> str: @@ -225,6 +236,14 @@ def off_surface_statement(measurement: OffSurfaceMeasurement) -> str: headline = "off-surface indicators: 0" lines = [headline] for indicator in INDICATOR_ORDER: + # An indicator that could not be evaluated says so in its own position. A bare zero + # here would read as "checked and clean", which is the opposite of unknown. + if indicator == WRITE_WITHOUT_WRITE_CALL and not measurement.mutation_intent_available: + lines.append( + f" {INDICATOR_LABELS[indicator]}: not evaluated — this file declares no task tags, " + "so mutation intent is unknown" + ) + continue addresses = measurement.addresses(indicator) suffix = f" [{', '.join(addresses)}]" if addresses else "" rule = f"; rule: {LOW_CALL_RULE}" if indicator == IMPLAUSIBLY_FEW_CALLS else "" diff --git a/evals/report/summary.py b/evals/report/summary.py index 652531cb..600c2c1d 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -8,6 +8,7 @@ from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, TaskResult from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family +from evals.task_metadata import TaskMetadata, entry_needs, task_metadata_from_rows from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import OffSurfaceMeasurement, measure_off_surface @@ -201,6 +202,7 @@ def summarize( *, expected_rows: int | None = None, run_keys: RunKeyValidation | None = None, + task_metadata: TaskMetadata | None = None, ) -> Summary: """Aggregate per-task metrics. @@ -209,6 +211,11 @@ def summarize( ``error`` rows remain harness errors (excluded from success, counted in ``harness_err``). """ + # The run's own task facts, so skip expectations and mutation intent describe what ran + # rather than what the working tree happens to say now. A caller that already filtered + # the meta header out of ``rows`` passes it explicitly. + if task_metadata is None: + task_metadata = task_metadata_from_rows(rows) by_task: dict[str, list[TaskResult]] = defaultdict(list) harness_errors_by_task: dict[str, int] = defaultdict(int) infrastructure_errors_by_task: dict[str, int] = defaultdict(int) @@ -248,7 +255,11 @@ def summarize( if row.skipped: family = skip_reason_family(row.skipped) skipped_task_reasons[row.skipped].add(task_id) - if is_expected_environment_capability_skip(row.skipped, task_id=task_id): + if is_expected_environment_capability_skip( + row.skipped, + task_id=task_id, + task_needs=entry_needs(task_metadata.get(task_id)) if task_metadata else None, + ): expected_skips += 1 expected_skip_reasons[family] += 1 completed_rows += 1 @@ -372,6 +383,6 @@ def summarize( unexpected_run_keys=run_keys.unexpected if run_keys is not None else (), multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), - off_surface=measure_off_surface(rows), + off_surface=measure_off_surface(rows, task_catalog=task_metadata or None), schema_friction=measure_schema_friction(rows), ) diff --git a/evals/report/table.py b/evals/report/table.py index bf37c28d..453d130d 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -7,7 +7,7 @@ from typing import Any from evals.results import TaskResult -from evals.tasks import TASKS_BY_ID +from evals.task_metadata import TaskMetadata, entry_prompt, task_metadata_from_rows from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import off_surface_statement @@ -224,6 +224,11 @@ def build_multi_surface_table( columns: list[str] = [] rows_by_column: dict[str, dict[str, list[TaskResult]]] = {} multiple_repetitions_by_column: dict[str, bool] = {} + # Prompt excerpts and mutation intent come from the runs being rendered, so collect the + # metadata every input file declared before the meta rows are filtered out below. + task_metadata: dict[str, dict[str, Any]] = {} + for _label, rows in file_rows: + task_metadata.update(task_metadata_from_rows(rows)) for label, rows in file_rows: columns.append(label) column_rows: dict[str, list[TaskResult]] = defaultdict(list) @@ -278,6 +283,7 @@ def build_multi_surface_table( [row for task_rows in rows_by_column[column].values() for row in task_rows], expected_rows=(expected_rows_by_column or {}).get(column), run_keys=(run_keys_by_column or {}).get(column), + task_metadata=task_metadata, ) footer[column] = { "success": successes, @@ -306,11 +312,17 @@ def build_multi_surface_table( "footer": footer, "multi_rep": multiple_repetitions, "multi_rep_by_col": multiple_repetitions_by_column, + "task_metadata": task_metadata, } -def prompt_excerpt(task_id: str) -> str: - prompt = (TASKS_BY_ID.get(task_id, {}).get("prompt") or "").replace("{project}", "P") +def prompt_excerpt(task_id: str, task_metadata: TaskMetadata | None = None) -> str: + """Render a short prompt excerpt from the run's own metadata. + + Empty when the file predates the persisted header: an excerpt taken from the current + checkout can describe a prompt the run never used. + """ + prompt = entry_prompt((task_metadata or {}).get(task_id)).replace("{project}", "P") return (prompt[:32] + "…") if len(prompt) > 32 else prompt @@ -318,6 +330,7 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) """Render multi-surface table as plain text or GitHub markdown.""" columns: list[str] = table["columns"] task_ids: list[str] = table["task_ids"] + task_metadata: TaskMetadata = table.get("task_metadata") or {} cells: dict[str, dict[str, str]] = table["cells"] footer: dict[str, dict[str, Any]] = table["footer"] multiple_repetitions = bool(table.get("multi_rep")) @@ -330,7 +343,7 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) lines.append(separator) for task_id in task_ids: row_cells = " | ".join(cells[task_id].get(column, "—") for column in columns) - lines.append(f"| {task_id} | {prompt_excerpt(task_id)} | {row_cells} |") + lines.append(f"| {task_id} | {prompt_excerpt(task_id, task_metadata)} | {row_cells} |") # Footer footer_parts = [] for column in columns: @@ -381,7 +394,7 @@ def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) lines.append(heading) lines.append("-" * len(heading)) for task_id in task_ids: - line = f"{task_id:5} {prompt_excerpt(task_id):34} " + line = f"{task_id:5} {prompt_excerpt(task_id, task_metadata):34} " for column in columns: line += f"{cells[task_id].get(column, '—'):{column_width}} " lines.append(line.rstrip()) diff --git a/evals/runner/live.py b/evals/runner/live.py index 38083ecc..7cdf854e 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -23,6 +23,7 @@ from evals.seed import make_plane_client, seed, teardown from evals.seed.identities import capture_seed_artifacts from evals.server_env import stdio_server_env +from evals.task_metadata import build_task_metadata from evals.tasks.catalog import battery_fingerprint, task_author, task_fingerprint from evals.tasks.prompts import PromptBindError, format_task_prompt @@ -594,6 +595,7 @@ async def run_live( expected_rows=total_runs, expected_task_ids=[str(task["id"]) for task in tasks], expected_reps=reps, + task_metadata=build_task_metadata(tasks), ) if maybe_write_run_meta(out_path, meta): print(f"wrote meta header battery={battery} label={label}", flush=True) diff --git a/evals/runner/meta.py b/evals/runner/meta.py index 089d81e5..35366c6f 100644 --- a/evals/runner/meta.py +++ b/evals/runner/meta.py @@ -9,6 +9,7 @@ from typing import Any from evals.results import RESULT_SCHEMA_VERSION +from evals.task_metadata import METADATA_FIELD, normalize_task_metadata def read_git_revision() -> str: @@ -49,6 +50,7 @@ def make_run_meta_row( expected_rows: int | None = None, expected_task_ids: list[str] | tuple[str, ...] | None = None, expected_reps: int | None = None, + task_metadata: dict[str, Any] | None = None, ts: str | None = None, ) -> dict[str, Any]: """Build the single first-line meta record for a new output JSONL.""" @@ -85,6 +87,9 @@ def make_run_meta_row( row["expected_task_ids"] = task_ids if expected_reps is not None: row["expected_reps"] = expected_reps + if task_metadata: + # Persisted so a report describes the run it reads rather than today's catalog. + row[METADATA_FIELD] = normalize_task_metadata(task_metadata) return row diff --git a/evals/skip_taxonomy.py b/evals/skip_taxonomy.py index 18eba332..728a3266 100644 --- a/evals/skip_taxonomy.py +++ b/evals/skip_taxonomy.py @@ -10,6 +10,7 @@ from __future__ import annotations +from collections.abc import Iterable from typing import Literal SkipDisposition = Literal["expected-capability", "dirty-environment", "unexpected"] @@ -50,11 +51,15 @@ def _plan_gated_capability(reason: str) -> str | None: return capability if capability in PLAN_GATED_CAPABILITIES else None -def _task_expected_capability_reasons(task_id: str) -> frozenset[str]: - from evals.tasks import TASKS_BY_ID +def _task_expected_capability_reasons(task_id: str, task_needs: Iterable[str] | None = None) -> frozenset[str]: + if task_needs is None: + # No run metadata: fall back to the checkout. Only reached for files written before + # the meta header carried each task's needs. + from evals.tasks import TASKS_BY_ID - task = TASKS_BY_ID.get(task_id) - needs = set(task.get("needs") or ()) if task is not None else set() + task = TASKS_BY_ID.get(task_id) + task_needs = task.get("needs") or () if task is not None else () + needs = set(str(need) for need in task_needs) reasons = { f"{PLAN_GATED_PREFIX}{capability}" for need, capability in _PLAN_GATED_CAPABILITY_BY_NEED.items() @@ -65,19 +70,29 @@ def _task_expected_capability_reasons(task_id: str) -> frozenset[str]: return frozenset(reasons) -def classify_skip_reason(reason: str, *, task_id: str | None = None) -> SkipDisposition: +def classify_skip_reason( + reason: str, + *, + task_id: str | None = None, + task_needs: Iterable[str] | None = None, +) -> SkipDisposition: """Classify a known capability skip, dirty environment, or unknown reason.""" is_known_capability = _plan_gated_capability(reason) is not None or reason == NO_ACTIVITY_WORKER_REASON - if is_known_capability and (task_id is None or reason in _task_expected_capability_reasons(task_id)): + if is_known_capability and (task_id is None or reason in _task_expected_capability_reasons(task_id, task_needs)): return "expected-capability" if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): return "dirty-environment" return "unexpected" -def is_expected_environment_capability_skip(reason: str, *, task_id: str | None = None) -> bool: +def is_expected_environment_capability_skip( + reason: str, + *, + task_id: str | None = None, + task_needs: Iterable[str] | None = None, +) -> bool: """Return whether a known absent environment capability caused the skip.""" - return classify_skip_reason(reason, task_id=task_id) == "expected-capability" + return classify_skip_reason(reason, task_id=task_id, task_needs=task_needs) == "expected-capability" def skip_reason_family(reason: str) -> str: diff --git a/evals/task_metadata.py b/evals/task_metadata.py new file mode 100644 index 00000000..61ff25ae --- /dev/null +++ b/evals/task_metadata.py @@ -0,0 +1,97 @@ +"""The task facts a report needs, persisted with the run instead of read from the checkout. + +Reports derived three things from the live catalog: whether a task mutates Plane, its prompt +text, and the fixtures it needs (which decides whether a plan-gated skip was expected). All +three are properties of *the run that was executed*, so reading them from the working tree +meant a result file could be reinterpreted after the catalog changed — the one thing the +battery fingerprint and identity validation exist to prevent. + +The run writes them into its meta header. A file that predates the header has no metadata, +and the reader says so rather than quietly substituting today's catalog. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +TaskMetadata = Mapping[str, Mapping[str, Any]] + +METADATA_FIELD = "task_metadata" +MUTATION_TAGS = frozenset({"write"}) + + +def build_task_metadata(tasks: Iterable[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + """Capture the report-relevant facts of the tasks this run is about to execute.""" + metadata: dict[str, dict[str, Any]] = {} + for task in tasks: + task_id = str(task.get("id") or "") + if not task_id: + continue + metadata[task_id] = { + "tags": sorted(str(tag) for tag in (task.get("tags") or ())), + "needs": sorted(str(need) for need in (task.get("needs") or ())), + "prompt": str(task.get("prompt") or ""), + } + return metadata + + +def normalize_task_metadata(value: Any) -> dict[str, dict[str, Any]]: + """Validate a persisted metadata map, dropping entries that cannot be trusted.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, dict[str, Any]] = {} + for raw_id, raw_entry in value.items(): + task_id = str(raw_id or "").strip() + if not task_id or not isinstance(raw_entry, Mapping): + continue + normalized[task_id] = { + "tags": sorted(str(tag) for tag in (raw_entry.get("tags") or ()) if str(tag)), + "needs": sorted(str(need) for need in (raw_entry.get("needs") or ()) if str(need)), + "prompt": str(raw_entry.get("prompt") or ""), + } + return normalized + + +def task_metadata_from_rows(rows: Iterable[Any]) -> dict[str, dict[str, Any]]: + """Merge the metadata declared by every meta header in the loaded rows.""" + merged: dict[str, dict[str, Any]] = {} + for row in rows: + if not isinstance(row, Mapping) or row.get("row_type") != "meta": + continue + merged.update(normalize_task_metadata(row.get(METADATA_FIELD))) + return merged + + +def entry_requires_mutation(entry: Mapping[str, Any] | None) -> bool: + """Whether a task was expected to change Plane state, from its persisted tags.""" + if not isinstance(entry, Mapping): + return False + return bool(MUTATION_TAGS.intersection(str(tag) for tag in (entry.get("tags") or ()))) + + +def entry_needs(entry: Mapping[str, Any] | None) -> tuple[str, ...]: + """The fixtures a task declared, from its persisted needs.""" + if not isinstance(entry, Mapping): + return () + return tuple(str(need) for need in (entry.get("needs") or ()) if str(need)) + + +def entry_prompt(entry: Mapping[str, Any] | None) -> str: + """The prompt text a task ran with, from its persisted metadata.""" + if not isinstance(entry, Mapping): + return "" + return str(entry.get("prompt") or "") + + +__all__ = [ + "METADATA_FIELD", + "MUTATION_TAGS", + "TaskMetadata", + "build_task_metadata", + "entry_needs", + "entry_prompt", + "entry_requires_mutation", + "normalize_task_metadata", + "task_metadata_from_rows", +] diff --git a/tests/evals/report/test_off_surface.py b/tests/evals/report/test_off_surface.py index 1e07c271..e4060a62 100644 --- a/tests/evals/report/test_off_surface.py +++ b/tests/evals/report/test_off_surface.py @@ -141,7 +141,10 @@ def test_reports_print_explicit_zero_addresses_rule_and_limitation(capsys): assert "RUN COMPLETE:" in single_output bypass = _row("W1", rep=3, num_calls=0, calls=[]) - comparison = ab_compare([clean], [bypass]) + # Mutation intent is a fact about the run, so the file carries it. Without the header a + # hand-built row has no tags and the write indicator is correctly silent. + write_meta = {"row_type": "meta", "task_metadata": {"W1": {"tags": ["write"]}}} + comparison = ab_compare([write_meta, clean], [write_meta, bypass]) print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) ab_output = capsys.readouterr().out diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index ddd1f4e8..55d9e669 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -392,7 +392,8 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" "off-surface indicators: 0\n" " zero-call success: 0\n" - " write without a write call: 0\n" + " write without a write call: not evaluated — this file declares no task tags, " + "so mutation intent is unknown\n" " answer without provenance: 0\n" " implausibly few calls: 0; rule: among at least 5 successful trace-usable repetitions for the same " "task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 6213d98a..718f28dc 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -330,7 +330,13 @@ def test_multi_surface_behaviours(case): def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): - rows = [_synth_row("R1", label="local", success=True, num_calls=2)] + # The prompt excerpt is read from the run's own metadata, never from the checkout, so a + # rendered table describes the prompt that actually ran. + meta = { + "row_type": "meta", + "task_metadata": {"R1": {"prompt": "In project {project}, what is the current state of the item?"}}, + } + rows = [meta, _synth_row("R1", label="local", success=True, num_calls=2)] rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) @@ -360,8 +366,9 @@ def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): def test_multi_surface_table_reports_off_surface_indicators_per_column_in_plain_and_markdown(): + write_meta = {"row_type": "meta", "task_metadata": {"W1": {"tags": ["write"]}}} clean = [_synth_row("R1", label="clean", num_calls=1, calls=[{"tool": "list_work_items"}])] - bypass = [_synth_row("W1", label="bypass", num_calls=0, calls=[])] + bypass = [write_meta, _synth_row("W1", label="bypass", num_calls=0, calls=[])] table = build_multi_surface_table([("clean", clean), ("bypass", bypass)]) plain = render_multi_surface_table(table) From 27d589f9cd9cf7fe4184ed775b38eacd51ca81a5 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 21:11:38 +0530 Subject: [PATCH 63/93] Give each driver surface a base its vendors implement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drivers/driver.py held both the owned API loop and the CLI subprocess template. Nothing crossed between them — measured, zero shared names of its own, only stdlib and the two leaf modules evidence.py and results.py — so the two halves were sharing a file and nothing else. Each surface now names its shared abstraction the same way: base.py holds what the sibling vendor files implement. In api/ that is the renamed backend.py, the protocol anthropic.py and openai.py satisfy; in cli/ it is the CliDriver template the four agent CLIs fill in. ApiDriver keeps its own module because it is the single concrete loop, not a base. The api-local BackendFactory alias is gone. It named a different shape than the registry's BackendFactory in the same package, and the move put the two side by side; it had one use, now inlined. process.py, sidecar.py and base.py stay asymmetric between the two directories on purpose. A CLI driver never speaks to a model, so it needs no backend; the API driver owns its loop and sees every call directly, so it needs neither a recording proxy nor a process group. --- evals/DESIGN.md | 5 +- evals/drivers/__init__.py | 3 +- evals/drivers/api/__init__.py | 2 +- evals/drivers/api/anthropic.py | 2 +- evals/drivers/api/{backend.py => base.py} | 0 evals/drivers/{ => api}/driver.py | 373 +-------------------- evals/drivers/api/openai.py | 2 +- evals/drivers/cli/antigravity.py | 2 +- evals/drivers/cli/base.py | 387 ++++++++++++++++++++++ evals/drivers/cli/claude.py | 2 +- evals/drivers/cli/codex.py | 2 +- evals/drivers/cli/opencode.py | 2 +- tests/evals/drivers/test_api_driver.py | 12 +- tests/evals/drivers/test_cli_driver.py | 4 +- 14 files changed, 418 insertions(+), 380 deletions(-) rename evals/drivers/api/{backend.py => base.py} (100%) rename evals/drivers/{ => api}/driver.py (56%) create mode 100644 evals/drivers/cli/base.py diff --git a/evals/DESIGN.md b/evals/DESIGN.md index e6851da0..83797ca1 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -329,12 +329,13 @@ evals/ debias.py I1-I5 and L1-L5 tasks and verifiers drivers/ __init__.py public exports and driver registry - driver.py AgentDriver seam, the API loop, and the CLI template api/ - backend.py neutral backend protocol and turn/tool dataclasses + base.py neutral backend protocol and turn/tool dataclasses + driver.py the owned model/tool loop anthropic.py Anthropic Messages translation openai.py OpenAI Chat Completions translation cli/ + base.py the subprocess template vendors fill in process.py subprocess lifecycle sidecar.py recording-proxy command and sidecar handling claude.py Claude Code CLI driver diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index 4f143bd0..acbdfbdb 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -8,11 +8,13 @@ from typing import Any +from evals.drivers.api.driver import ApiDriver from evals.drivers.cli.antigravity import ( AntigravityCliDriver, prepare_antigravity_fake_home, write_antigravity_mcp_config, ) +from evals.drivers.cli.base import CliDriver, CliRunError from evals.drivers.cli.claude import ( ClaudeCliDriver, find_claude_transcript, @@ -48,7 +50,6 @@ proxy_wrap_server_command, wait_for_proxy_meta, ) -from evals.drivers.driver import ApiDriver, CliDriver, CliRunError # Registry # --------------------------------------------------------------------------- diff --git a/evals/drivers/api/__init__.py b/evals/drivers/api/__init__.py index 1299e26a..68ac54f0 100644 --- a/evals/drivers/api/__init__.py +++ b/evals/drivers/api/__init__.py @@ -3,7 +3,7 @@ # Import built-in adapters for their registrations. Each adapter owns its SDK # translation and keeps its optional SDK import lazy until construction. from evals.drivers.api.anthropic import AnthropicBackend -from evals.drivers.api.backend import ( +from evals.drivers.api.base import ( BACKEND_REGISTRY, KNOWN_API_PROVIDERS, MODEL_TIERS, diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py index ee137c01..448f7ab6 100644 --- a/evals/drivers/api/anthropic.py +++ b/evals/drivers/api/anthropic.py @@ -4,7 +4,7 @@ from typing import Any -from evals.drivers.api.backend import ( +from evals.drivers.api.base import ( StopReason, ToolCall, ToolResult, diff --git a/evals/drivers/api/backend.py b/evals/drivers/api/base.py similarity index 100% rename from evals/drivers/api/backend.py rename to evals/drivers/api/base.py diff --git a/evals/drivers/driver.py b/evals/drivers/api/driver.py similarity index 56% rename from evals/drivers/driver.py rename to evals/drivers/api/driver.py index 9200f1a2..dff1c2be 100644 --- a/evals/drivers/driver.py +++ b/evals/drivers/api/driver.py @@ -1,26 +1,29 @@ -"""Owned API and subprocess-backed CLI evaluation drivers.""" +"""The owned model/tool loop: run one task through a registered API backend. + +Unlike a CLI driver, this driver *is* the loop — it lists the MCP tool surface, asks the +backend for a turn, executes the calls itself and records each result as it comes back. So +it needs no recording proxy and no subprocess supervision: the evidence is in hand. + +Provider differences live behind ``ModelBackend`` in ``backend.py``; nothing here knows +which vendor answered. +""" from __future__ import annotations import asyncio import inspect import json -import subprocess import sys -import tempfile import time -from abc import ABC, abstractmethod from collections.abc import Callable from contextlib import asynccontextmanager -from dataclasses import dataclass, field from pathlib import Path from typing import Any from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client -from evals import REPO_ROOT -from evals.drivers.api.backend import ( +from evals.drivers.api.base import ( KNOWN_API_PROVIDERS, ModelBackend, StopReason, @@ -28,15 +31,6 @@ ToolSpec, create_backend, ) -from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess -from evals.drivers.cli.sidecar import ( - ProxySidecarResult, - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - load_proxy_sidecar, - proxy_wrap_server_command, -) from evals.evidence import ( configured_evidence_labels, normalize_evidence_aggregates, @@ -45,7 +39,6 @@ observed_aggregate_labels, observed_aggregates, observed_sentinel_labels, - write_evidence_config, ) from evals.results import AgentRun, Usage from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens @@ -53,7 +46,6 @@ DEFAULT_MAX_TOKENS = 8192 -BackendFactory = Callable[[str, int], ModelBackend] McpSessionFactory = Callable[[StdioServerParameters], Any] @@ -133,7 +125,7 @@ def __init__( *, provider: str = "anthropic", client: Any | None = None, - backend_factory: BackendFactory | None = None, + backend_factory: Callable[[str, int], ModelBackend] | None = None, mcp_session_factory: McpSessionFactory | None = None, server_command: list[str] | None = None, python_bin: str | None = None, @@ -447,352 +439,9 @@ async def _run_task( ) -@dataclass -class CliLaunch: - """Vendor-prepared CLI launch details.""" - - cwd: Path - config_args: list[str] = field(default_factory=list) - env: dict[str, str] | None = None - artifact_dir: Path | None = None - - -@dataclass -class CliOutput: - """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" - - final_text: str - calls: list[dict[str, Any]] = field(default_factory=list) - client_tool_calls: list[dict[str, Any]] = field(default_factory=list) - usage: dict[str, Any] | None = None - usage_total: dict[str, Any] | None = None - stopped_reason: str = "end_turn" - raw_ref: str | None = None - call_source: str = "json" - hit_max_turns: bool = False - - -class CliOutputError(RuntimeError): - """Signal that vendor output could not produce a valid ``AgentRun``.""" - - -class CliRunError(RuntimeError): - """CLI failure retaining typed sidecar observations for the result row.""" - - def __init__(self, message: str, sidecar: ProxySidecarResult | None = None) -> None: - super().__init__(message) - self.trace_integrity = sidecar.trace_integrity if sidecar is not None else True - self.trace_integrity_reason = sidecar.trace_integrity_reason if sidecar is not None else None - self.tool_manifest_fingerprint = sidecar.tool_manifest_fingerprint if sidecar is not None else None - - -class CliDriver(ABC): - """Template for CLI drivers that run one MCP-backed subprocess task.""" - - name: str - experimental = False - default_call_source = "json" - run_notes: tuple[str, ...] = () - temp_dir_prefix = "plane-eval-cli-" - temp_dir_in_cwd = False - exit_note_prefix: str | None = None - include_stderr_in_exit_note = False - - def __init__( - self, - *, - python_bin: str | None = None, - runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, - server_command: list[str] | None = None, - use_proxy: bool = True, - record_result_payloads: bool = False, - ) -> None: - self.python_bin = python_bin or sys.executable - self._runner = runner or run_cli_subprocess - self.server_command = list(server_command) if server_command else None - self.use_proxy = use_proxy - self.record_result_payloads = record_result_payloads - - def validate_run(self) -> None: - """Reject a launch before any temporary state is created, if needed.""" - return None - - @abstractmethod - def write_mcp_config( - self, - temp_dir: Path, - *, - task_cwd: Path, - server_command: list[str], - child_env: dict[str, str], - ) -> CliLaunch: - """Write vendor MCP configuration and return launch settings.""" - - @abstractmethod - def build_command( - self, - prompt: str, - *, - model: str | None, - max_turns: int, - system: str | None, - launch: CliLaunch, - ) -> list[str]: - """Build the vendor CLI command.""" - - def invoke_cli( - self, - command: list[str], - *, - launch: CliLaunch, - timeout_s: int, - ) -> subprocess.CompletedProcess[str]: - """Invoke the configured runner with the shared subprocess contract.""" - kwargs: dict[str, Any] = { - "cwd": str(launch.cwd), - "capture_output": True, - "text": True, - "timeout": timeout_s, - } - if launch.env is not None: - kwargs["env"] = launch.env - return self._runner(command, **kwargs) - - @abstractmethod - def parse_output( - self, - proc: subprocess.CompletedProcess[str], - *, - launch: CliLaunch, - task_cwd: Path, - max_turns: int, - notes: list[str], - ) -> CliOutput: - """Parse vendor output into the normalized CLI result shape.""" - - def finalize_run( - self, - proc: subprocess.CompletedProcess[str], - *, - output: CliOutput, - notes: list[str], - ) -> None: - """Apply vendor handling that must occur after proxy reconciliation.""" - del output - if proc.returncode != 0 and self.exit_note_prefix: - notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") - stderr = proc.stderr or "" - if self.include_stderr_in_exit_note and stderr.strip(): - notes.append(stderr.strip()[:500]) - - def run_task( - self, - prompt: str, - mcp_env: dict[str, str], - model: str | None, - max_turns: int, - *, - system: str | None = None, - cwd: Path | None = None, - evidence_sentinels: dict[str, Any] | None = None, - evidence_targets: dict[str, Any] | None = None, - evidence_aggregates: dict[str, Any] | None = None, - artifact_dir: Path | None = None, - ) -> AgentRun: - """Run one CLI task using the shared configuration/proxy/timeout flow.""" - task_cwd = (cwd or REPO_ROOT).resolve() - notes = list(self.run_notes) - self.validate_run() - temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None - - with ( - tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td, - tempfile.TemporaryDirectory(prefix="plane-eval-evidence-") as evidence_td, - ): - temp_dir = Path(td) - sidecar = temp_dir / "proxy-sidecar.jsonl" - child_env = { - key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") - } - evidence = normalize_evidence_sentinels(evidence_sentinels) - targets = normalize_evidence_targets(evidence_targets) - aggregates = normalize_evidence_aggregates(evidence_aggregates) - evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) - - def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: - """Turn observed proxy values into labels using harness-held seed truth.""" - for call in calls: - labels = set(call.get("observed_sentinels") or []) - labels.update(observed_aggregate_labels(call.get("observed_aggregates"), aggregates)) - if evidence_active: - call["observed_sentinels"] = sorted(labels) - - real_command = ( - list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] - ) - server_command = real_command - if self.use_proxy: - evidence_path = None - if evidence_active: - evidence_path = Path(evidence_td) / "proxy-evidence.json" - write_evidence_config(evidence_path, evidence, targets, aggregates) - server_command = proxy_wrap_server_command( - real_command, - sidecar_path=sidecar, - python_bin=self.python_bin, - record_result_payloads=self.record_result_payloads, - evidence_path=evidence_path, - ) - child_env = ensure_proxy_pythonpath(child_env) - - launch = self.write_mcp_config( - temp_dir, - task_cwd=task_cwd, - server_command=server_command, - child_env=child_env, - ) - launch.artifact_dir = ( - artifact_dir - if artifact_dir is not None - else task_cwd / "evals" / "output" / "driver-artifacts" / self.name - ).resolve() - command = self.build_command( - prompt, - model=model, - max_turns=max_turns, - system=system, - launch=launch, - ) - timeout_s = max(120, max_turns * 60) - # Persisted schema v1 defines wall time as the CLI invocation only. - started_at = time.perf_counter() - - try: - proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) - except subprocess.TimeoutExpired as exc: - wall = time.perf_counter() - started_at - notes.append(f"timeout after {timeout_s}s") - note_timeout_kill(notes, exc) - calls: list[dict[str, Any]] = [] - client_calls: list[dict[str, Any]] = [] - call_source = self.default_call_source - trace_integrity = True - trace_integrity_reason = None - tool_manifest_fingerprint = None - if self.use_proxy: - sidecar_result = harvest_proxy_after_cli_timeout( - calls, - client_calls, - sidecar, - notes, - ) - calls, client_calls, call_source = sidecar_result - verify_aggregate_observations(calls) - trace_integrity = sidecar_result.trace_integrity - trace_integrity_reason = sidecar_result.trace_integrity_reason - tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint - evidence_available = False - if evidence_active and call_source == "proxy": - _proxy_calls, status = load_proxy_sidecar(sidecar) - evidence_available = bool( - status.get("state") == "complete" and status.get("evidence_trace_available") - ) - if not evidence_available: - notes.append("proxy_response_evidence_unavailable") - return AgentRun( - calls=calls, - client_tool_calls=client_calls, - final_text="", - usage=None, - stopped_reason="timeout", - raw_ref=None, - usage_scope="run", - call_source=call_source, - hit_max_turns=False, - wall_time_s=round(wall, 3), - evidence_trace_available=evidence_available, - trace_integrity=trace_integrity, - trace_integrity_reason=trace_integrity_reason, - tool_manifest_fingerprint=tool_manifest_fingerprint, - experimental=self.experimental, - notes=notes, - ) - - wall = time.perf_counter() - started_at - try: - output = self.parse_output( - proc, - launch=launch, - task_cwd=task_cwd, - max_turns=max_turns, - notes=notes, - ) - except CliOutputError as exc: - sidecar_result = None - if self.use_proxy: - sidecar_result = apply_proxy_sidecar([], [], sidecar, notes) - detail = "; ".join(notes) - raise CliRunError(f"{exc}: {detail}", sidecar_result) from None - - trace_integrity = True - trace_integrity_reason = None - tool_manifest_fingerprint = None - if self.use_proxy: - sidecar_result = apply_proxy_sidecar( - output.calls, - output.client_tool_calls, - sidecar, - notes, - ) - calls, client_calls, proxy_source = sidecar_result - output.calls = calls - output.client_tool_calls = client_calls - verify_aggregate_observations(output.calls) - trace_integrity = sidecar_result.trace_integrity - trace_integrity_reason = sidecar_result.trace_integrity_reason - tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint - if proxy_source == "proxy": - output.call_source = "proxy" - - evidence_available = False - if evidence_active and output.call_source == "proxy": - _proxy_calls, status = load_proxy_sidecar(sidecar) - evidence_available = bool(status.get("state") == "complete" and status.get("evidence_trace_available")) - if not evidence_available: - notes.append("proxy_response_evidence_unavailable") - - self.finalize_run(proc, output=output, notes=notes) - return AgentRun( - calls=output.calls, - client_tool_calls=output.client_tool_calls, - final_text=output.final_text, - usage=output.usage, - usage_total=output.usage_total, - stopped_reason=output.stopped_reason, - raw_ref=output.raw_ref, - usage_scope="run", - call_source=output.call_source, - hit_max_turns=output.hit_max_turns, - wall_time_s=round(wall, 3), - evidence_trace_available=evidence_available, - trace_integrity=trace_integrity, - trace_integrity_reason=trace_integrity_reason, - tool_manifest_fingerprint=tool_manifest_fingerprint, - experimental=self.experimental, - notes=notes, - ) - - __all__ = [ "DEFAULT_MAX_TOKENS", - "KNOWN_API_PROVIDERS", "ApiDriver", - "BackendFactory", - "CliDriver", - "CliLaunch", - "CliOutput", - "CliOutputError", - "CliRunError", "McpSessionFactory", "tool_result_from_mcp", "tool_spec_from_mcp", diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py index a6e54630..f229afdf 100644 --- a/evals/drivers/api/openai.py +++ b/evals/drivers/api/openai.py @@ -5,7 +5,7 @@ import json from typing import Any -from evals.drivers.api.backend import ( +from evals.drivers.api.base import ( StopReason, ToolCall, ToolResult, diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index fa6a502d..1b83a910 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -8,7 +8,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput # Antigravity CLI (agy) — proxy-first # --------------------------------------------------------------------------- diff --git a/evals/drivers/cli/base.py b/evals/drivers/cli/base.py new file mode 100644 index 00000000..5d4c9776 --- /dev/null +++ b/evals/drivers/cli/base.py @@ -0,0 +1,387 @@ +"""The subprocess template every CLI vendor driver fills in. + +A CLI driver runs the agent the user already pays for, in a process the harness does not +control. So unlike the API driver it cannot see the tool calls happen: it wraps the MCP +server with the recording proxy (``sidecar``), supervises the process group +(``process``), and reconciles what the proxy captured against what the vendor reported. + +Vendors supply four things — the MCP config file, the command, the output parse, and any +post-reconciliation handling. Everything shared about the run lives here. +""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import time +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from evals import REPO_ROOT +from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( + ProxySidecarResult, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + proxy_wrap_server_command, +) +from evals.evidence import ( + configured_evidence_labels, + normalize_evidence_aggregates, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_aggregate_labels, + write_evidence_config, +) +from evals.results import AgentRun + + +@dataclass +class CliLaunch: + """Vendor-prepared CLI launch details.""" + + cwd: Path + config_args: list[str] = field(default_factory=list) + env: dict[str, str] | None = None + artifact_dir: Path | None = None + + +@dataclass +class CliOutput: + """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" + + final_text: str + calls: list[dict[str, Any]] = field(default_factory=list) + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + usage: dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + stopped_reason: str = "end_turn" + raw_ref: str | None = None + call_source: str = "json" + hit_max_turns: bool = False + + +class CliOutputError(RuntimeError): + """Signal that vendor output could not produce a valid ``AgentRun``.""" + + +class CliRunError(RuntimeError): + """CLI failure retaining typed sidecar observations for the result row.""" + + def __init__(self, message: str, sidecar: ProxySidecarResult | None = None) -> None: + super().__init__(message) + self.trace_integrity = sidecar.trace_integrity if sidecar is not None else True + self.trace_integrity_reason = sidecar.trace_integrity_reason if sidecar is not None else None + self.tool_manifest_fingerprint = sidecar.tool_manifest_fingerprint if sidecar is not None else None + + +class CliDriver(ABC): + """Template for CLI drivers that run one MCP-backed subprocess task.""" + + name: str + experimental = False + default_call_source = "json" + run_notes: tuple[str, ...] = () + temp_dir_prefix = "plane-eval-cli-" + temp_dir_in_cwd = False + exit_note_prefix: str | None = None + include_stderr_in_exit_note = False + + def __init__( + self, + *, + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads + + def validate_run(self) -> None: + """Reject a launch before any temporary state is created, if needed.""" + return None + + @abstractmethod + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + """Write vendor MCP configuration and return launch settings.""" + + @abstractmethod + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + """Build the vendor CLI command.""" + + def invoke_cli( + self, + command: list[str], + *, + launch: CliLaunch, + timeout_s: int, + ) -> subprocess.CompletedProcess[str]: + """Invoke the configured runner with the shared subprocess contract.""" + kwargs: dict[str, Any] = { + "cwd": str(launch.cwd), + "capture_output": True, + "text": True, + "timeout": timeout_s, + } + if launch.env is not None: + kwargs["env"] = launch.env + return self._runner(command, **kwargs) + + @abstractmethod + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + """Parse vendor output into the normalized CLI result shape.""" + + def finalize_run( + self, + proc: subprocess.CompletedProcess[str], + *, + output: CliOutput, + notes: list[str], + ) -> None: + """Apply vendor handling that must occur after proxy reconciliation.""" + del output + if proc.returncode != 0 and self.exit_note_prefix: + notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") + stderr = proc.stderr or "" + if self.include_stderr_in_exit_note and stderr.strip(): + notes.append(stderr.strip()[:500]) + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, + artifact_dir: Path | None = None, + ) -> AgentRun: + """Run one CLI task using the shared configuration/proxy/timeout flow.""" + task_cwd = (cwd or REPO_ROOT).resolve() + notes = list(self.run_notes) + self.validate_run() + temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None + + with ( + tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td, + tempfile.TemporaryDirectory(prefix="plane-eval-evidence-") as evidence_td, + ): + temp_dir = Path(td) + sidecar = temp_dir / "proxy-sidecar.jsonl" + child_env = { + key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") + } + evidence = normalize_evidence_sentinels(evidence_sentinels) + targets = normalize_evidence_targets(evidence_targets) + aggregates = normalize_evidence_aggregates(evidence_aggregates) + evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) + + def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: + """Turn observed proxy values into labels using harness-held seed truth.""" + for call in calls: + labels = set(call.get("observed_sentinels") or []) + labels.update(observed_aggregate_labels(call.get("observed_aggregates"), aggregates)) + if evidence_active: + call["observed_sentinels"] = sorted(labels) + + real_command = ( + list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] + ) + server_command = real_command + if self.use_proxy: + evidence_path = None + if evidence_active: + evidence_path = Path(evidence_td) / "proxy-evidence.json" + write_evidence_config(evidence_path, evidence, targets, aggregates) + server_command = proxy_wrap_server_command( + real_command, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + evidence_path=evidence_path, + ) + child_env = ensure_proxy_pythonpath(child_env) + + launch = self.write_mcp_config( + temp_dir, + task_cwd=task_cwd, + server_command=server_command, + child_env=child_env, + ) + launch.artifact_dir = ( + artifact_dir + if artifact_dir is not None + else task_cwd / "evals" / "output" / "driver-artifacts" / self.name + ).resolve() + command = self.build_command( + prompt, + model=model, + max_turns=max_turns, + system=system, + launch=launch, + ) + timeout_s = max(120, max_turns * 60) + # Persisted schema v1 defines wall time as the CLI invocation only. + started_at = time.perf_counter() + + try: + proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - started_at + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = self.default_call_source + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None + if self.use_proxy: + sidecar_result = harvest_proxy_after_cli_timeout( + calls, + client_calls, + sidecar, + notes, + ) + calls, client_calls, call_source = sidecar_result + verify_aggregate_observations(calls) + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint + evidence_available = False + if evidence_active and call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + evidence_available = bool( + status.get("state") == "complete" and status.get("evidence_trace_available") + ) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, + experimental=self.experimental, + notes=notes, + ) + + wall = time.perf_counter() - started_at + try: + output = self.parse_output( + proc, + launch=launch, + task_cwd=task_cwd, + max_turns=max_turns, + notes=notes, + ) + except CliOutputError as exc: + sidecar_result = None + if self.use_proxy: + sidecar_result = apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise CliRunError(f"{exc}: {detail}", sidecar_result) from None + + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None + if self.use_proxy: + sidecar_result = apply_proxy_sidecar( + output.calls, + output.client_tool_calls, + sidecar, + notes, + ) + calls, client_calls, proxy_source = sidecar_result + output.calls = calls + output.client_tool_calls = client_calls + verify_aggregate_observations(output.calls) + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint + if proxy_source == "proxy": + output.call_source = "proxy" + + evidence_available = False + if evidence_active and output.call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + evidence_available = bool(status.get("state") == "complete" and status.get("evidence_trace_available")) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") + + self.finalize_run(proc, output=output, notes=notes) + return AgentRun( + calls=output.calls, + client_tool_calls=output.client_tool_calls, + final_text=output.final_text, + usage=output.usage, + usage_total=output.usage_total, + stopped_reason=output.stopped_reason, + raw_ref=output.raw_ref, + usage_scope="run", + call_source=output.call_source, + hit_max_turns=output.hit_max_turns, + wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, + experimental=self.experimental, + notes=notes, + ) + + +__all__ = [ + "CliDriver", + "CliLaunch", + "CliOutput", + "CliOutputError", + "CliRunError", +] diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index 6cf2b231..f1d9927d 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Any -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError from evals.tool_names import normalize_tool_call, split_plane_and_client_calls diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index f5651a08..ca4d63c8 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -16,8 +16,8 @@ from pathlib import Path from typing import Any +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput from evals.drivers.cli.process import run_cli_subprocess -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput from evals.tool_names import normalize_tool_call, split_plane_and_client_calls diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py index ee39f3dd..af60f3f8 100644 --- a/evals/drivers/cli/opencode.py +++ b/evals/drivers/cli/opencode.py @@ -9,7 +9,7 @@ from collections.abc import Callable from pathlib import Path -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput # OpenCode CLI — proxy-first # --------------------------------------------------------------------------- diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 0f45ae63..4df4fbe8 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -374,8 +374,8 @@ async def __aexit__(self, *_args): await self.message_handler(SimpleNamespace(root=SimpleNamespace(method="notifications/tools/list_changed"))) return None - monkeypatch.setattr("evals.drivers.driver.stdio_client", fake_stdio_client) - monkeypatch.setattr("evals.drivers.driver.ClientSession", FakeClientSessionContext) + monkeypatch.setattr("evals.drivers.api.driver.stdio_client", fake_stdio_client) + monkeypatch.setattr("evals.drivers.api.driver.ClientSession", FakeClientSessionContext) driver = ApiDriver( provider="anthropic", backend_factory=lambda _model, _max_tokens: backend, @@ -835,7 +835,7 @@ def test_openai_backend_behaviours(case): def _tool_spec_from_mcp_reads_dict_and_object_entries(): - from evals.drivers.driver import tool_spec_from_mcp + from evals.drivers.api.driver import tool_spec_from_mcp as_dict = tool_spec_from_mcp( {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} @@ -851,7 +851,7 @@ def _tool_spec_from_mcp_reads_dict_and_object_entries(): def _tool_result_from_mcp_text_only_joins_blocks(): - from evals.drivers.driver import tool_result_from_mcp + from evals.drivers.api.driver import tool_result_from_mcp result = tool_result_from_mcp( "call_1", @@ -866,7 +866,7 @@ def _tool_result_from_mcp_text_only_joins_blocks(): def _tool_result_from_mcp_serializes_non_text_blocks(): - from evals.drivers.driver import tool_result_from_mcp + from evals.drivers.api.driver import tool_result_from_mcp mixed = tool_result_from_mcp( "call_2", @@ -881,7 +881,7 @@ def _tool_result_from_mcp_serializes_non_text_blocks(): def _tool_result_from_mcp_propagates_error_flag_in_both_spellings(): - from evals.drivers.driver import tool_result_from_mcp + from evals.drivers.api.driver import tool_result_from_mcp assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 592b7cfa..6568bf9c 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -30,7 +30,7 @@ run_cli_subprocess, wait_for_proxy_meta, ) -from evals.drivers.driver import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from tests.evals.conftest import case_params @@ -277,7 +277,7 @@ def short_timeout_runner(cmd, **kwargs): def _cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path, monkeypatch): clock = {"now": 0.0} - monkeypatch.setattr("evals.drivers.driver.time.perf_counter", lambda: clock["now"]) + monkeypatch.setattr("evals.drivers.cli.base.time.perf_counter", lambda: clock["now"]) class MinimalCliDriver(CliDriver): name = "minimal-cli" From 95b2010b502899c613b4a024213f08c9d148542f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 17 Aug 2026 21:18:01 +0530 Subject: [PATCH 64/93] Load only the driver surface the run asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals/drivers/__init__.py re-exported forty names. Two of them, KNOWN_DRIVERS and get_driver, are what production uses; the rest were vendor internals — find_codex_rollout, parse_claude_transcript_calls, proxy_pid_path — re-exported only so tests could reach them through the package. Because Python runs a package's __init__ before any submodule, that wall made every consumer load all five agent CLIs. Importing evals.drivers.api.base, which shares nothing with them, pulled in the whole CLI tree: 108 modules became 699. Splitting driver.py did not fix this on its own, and could not have. Each vendor is now imported inside the get_driver branch that returns it, and tests import each name from the module that defines it. Reading the registry loads neither surface; asking for codex-cli loads the CLI side and not the API one, nor the other three vendors. Five boundary cases pin that, in fresh interpreters. They need a second probe: the existing one reduces a module to its depth-1 package name, so both surfaces read as "drivers" and the edge is inexpressible. The new probe matches full module prefixes, and has its own positive control. --- evals/DESIGN.md | 2 +- evals/drivers/__init__.py | 108 +++++-------------------- tests/evals/drivers/test_api_driver.py | 2 +- tests/evals/drivers/test_cli_driver.py | 23 +++--- tests/evals/drivers/test_vendors.py | 31 ++++--- tests/evals/runner/test_live.py | 4 +- tests/evals/test_package_boundaries.py | 36 +++++++++ tests/evals/test_proxy.py | 2 +- tests/evals/test_results.py | 4 +- 9 files changed, 89 insertions(+), 123 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 83797ca1..80da24e7 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -328,7 +328,7 @@ evals/ cross.py C1-C2 tasks and verifiers debias.py I1-I5 and L1-L5 tasks and verifiers drivers/ - __init__.py public exports and driver registry + __init__.py the driver registry, loading only the surface it is asked for api/ base.py neutral backend protocol and turn/tool dataclasses driver.py the owned model/tool loop diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py index acbdfbdb..1de4cbe6 100644 --- a/evals/drivers/__init__.py +++ b/evals/drivers/__init__.py @@ -2,113 +2,49 @@ The ``api`` driver owns a provider-neutral loop; CLI drivers spawn locally installed agent CLIs on the user's own subscription. Probed CLI details live with each vendor. + +Only the registry lives here, and each driver is imported inside the branch that returns +it. This file used to re-export forty names, most of them vendor internals reached only by +tests — and because Python runs a package's ``__init__`` before any submodule, that wall +made *every* consumer load all five agent CLIs. Importing the API backend, which shares +nothing with them, pulled in the whole CLI tree. """ from __future__ import annotations -from typing import Any - -from evals.drivers.api.driver import ApiDriver -from evals.drivers.cli.antigravity import ( - AntigravityCliDriver, - prepare_antigravity_fake_home, - write_antigravity_mcp_config, -) -from evals.drivers.cli.base import CliDriver, CliRunError -from evals.drivers.cli.claude import ( - ClaudeCliDriver, - find_claude_transcript, - normalize_claude_usage, - parse_claude_json_result, - parse_claude_transcript_calls, - prepare_claude_isolated_environment, - write_claude_mcp_config, -) -from evals.drivers.cli.codex import ( - CodexCliDriver, - find_codex_rollout, - parse_codex_jsonl_events, - parse_codex_rollout_calls, - prepare_codex_home, - write_codex_mcp_config, - write_codex_mcp_override_args, -) -from evals.drivers.cli.opencode import ( - OpencodeCliDriver, - prepare_opencode_isolated_environment, - write_opencode_mcp_config, -) -from evals.drivers.cli.process import kill_process_group, note_timeout_kill, run_cli_subprocess -from evals.drivers.cli.sidecar import ( - apply_proxy_sidecar, - ensure_proxy_pythonpath, - harvest_proxy_after_cli_timeout, - load_proxy_sidecar, - load_proxy_sidecar_calls, - proxy_pid_path, - proxy_session_paths, - proxy_wrap_server_command, - wait_for_proxy_meta, -) +from typing import TYPE_CHECKING, Any -# Registry -# --------------------------------------------------------------------------- +if TYPE_CHECKING: + from evals.drivers.api.driver import ApiDriver + from evals.drivers.cli.base import CliDriver KNOWN_DRIVERS = frozenset({"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: - """Return a driver instance.""" + """Return a driver instance, loading only the surface it names.""" key = (name or "api").strip().lower() if key == "api": + from evals.drivers.api.driver import ApiDriver + return ApiDriver(**kwargs) if key == "claude-cli": + from evals.drivers.cli.claude import ClaudeCliDriver + return ClaudeCliDriver(**kwargs) if key == "codex-cli": + from evals.drivers.cli.codex import CodexCliDriver + return CodexCliDriver(**kwargs) if key == "antigravity-cli": + from evals.drivers.cli.antigravity import AntigravityCliDriver + return AntigravityCliDriver(**kwargs) if key == "opencode-cli": + from evals.drivers.cli.opencode import OpencodeCliDriver + return OpencodeCliDriver(**kwargs) raise ValueError(f"unknown driver {name!r}; expected one of {sorted(KNOWN_DRIVERS)}") -__all__ = [ - "KNOWN_DRIVERS", - "AntigravityCliDriver", - "ApiDriver", - "ClaudeCliDriver", - "CliDriver", - "CliRunError", - "CodexCliDriver", - "OpencodeCliDriver", - "apply_proxy_sidecar", - "ensure_proxy_pythonpath", - "find_claude_transcript", - "find_codex_rollout", - "get_driver", - "harvest_proxy_after_cli_timeout", - "kill_process_group", - "load_proxy_sidecar", - "load_proxy_sidecar_calls", - "normalize_claude_usage", - "note_timeout_kill", - "parse_claude_json_result", - "parse_claude_transcript_calls", - "parse_codex_jsonl_events", - "parse_codex_rollout_calls", - "prepare_antigravity_fake_home", - "prepare_claude_isolated_environment", - "prepare_codex_home", - "prepare_opencode_isolated_environment", - "proxy_pid_path", - "proxy_session_paths", - "proxy_wrap_server_command", - "run_cli_subprocess", - "wait_for_proxy_meta", - "write_antigravity_mcp_config", - "write_claude_mcp_config", - "write_codex_mcp_override_args", - "write_codex_mcp_config", - "write_opencode_mcp_config", -] +__all__ = ["KNOWN_DRIVERS", "get_driver"] diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 4df4fbe8..3bd50fac 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -10,7 +10,6 @@ import pytest -from evals.drivers import ApiDriver from evals.drivers.api import ( KNOWN_API_PROVIDERS, AnthropicBackend, @@ -26,6 +25,7 @@ resolve_backend_model, unregister_backend, ) +from evals.drivers.api.driver import ApiDriver from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.token_counting import estimate_result_tokens from evals.tool_manifest import tool_manifest_fingerprint diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 6568bf9c..6fc8a051 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -14,23 +14,22 @@ import pytest import tomllib -from evals.drivers import ( - AntigravityCliDriver, - ClaudeCliDriver, - CodexCliDriver, - OpencodeCliDriver, +from evals.drivers.cli.antigravity import AntigravityCliDriver +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.cli.claude import ClaudeCliDriver +from evals.drivers.cli.codex import CodexCliDriver, prepare_codex_home +from evals.drivers.cli.opencode import OpencodeCliDriver +from evals.drivers.cli.process import run_cli_subprocess +from evals.drivers.cli.sidecar import ( apply_proxy_sidecar, ensure_proxy_pythonpath, harvest_proxy_after_cli_timeout, load_proxy_sidecar, load_proxy_sidecar_calls, - prepare_codex_home, proxy_pid_path, proxy_wrap_server_command, - run_cli_subprocess, wait_for_proxy_meta, ) -from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from tests.evals.conftest import case_params @@ -96,7 +95,7 @@ def _run_cli_subprocess_kills_process_group_on_timeout(tmp_path, _monkeypatch): def _run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): - import evals.drivers as drivers_mod + from evals.drivers.cli import process as process_mod pidfile = tmp_path / "pids.txt" script = tmp_path / "sticky.py" @@ -153,7 +152,7 @@ def boom_communicate(self, *a, **k): alive = [p for p in pids if _pid_alive(p)] assert not alive, f"group survived BaseException path: {alive}" # silence unused import lint if any - assert drivers_mod.run_cli_subprocess is run_cli_subprocess + assert process_mod.run_cli_subprocess is run_cli_subprocess @pytest.mark.parametrize( @@ -172,7 +171,7 @@ def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): import signal from types import SimpleNamespace - from evals.drivers import kill_process_group + from evals.drivers.cli.process import kill_process_group pidfile = tmp_path / "pids.txt" script = tmp_path / "sticky_leader.py" @@ -253,7 +252,7 @@ def _cli_driver_timeout_notes_process_group_kill(tmp_path, _monkeypatch): ) # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. - from evals.drivers import run_cli_subprocess as real_runner + from evals.drivers.cli.process import run_cli_subprocess as real_runner def short_timeout_runner(cmd, **kwargs): kwargs = dict(kwargs) diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index ff84fd28..e4838492 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -10,23 +10,22 @@ import pytest -from evals.drivers import ( - KNOWN_DRIVERS, +from evals.drivers import KNOWN_DRIVERS, get_driver +from evals.drivers.api.driver import ApiDriver +from evals.drivers.cli.antigravity import ( AntigravityCliDriver, - ApiDriver, + prepare_antigravity_fake_home, + write_antigravity_mcp_config, +) +from evals.drivers.cli.claude import ( ClaudeCliDriver, - CodexCliDriver, - OpencodeCliDriver, - get_driver, normalize_claude_usage, parse_claude_json_result, parse_claude_transcript_calls, - parse_codex_jsonl_events, - prepare_antigravity_fake_home, - write_antigravity_mcp_config, write_claude_mcp_config, - write_opencode_mcp_config, ) +from evals.drivers.cli.codex import CodexCliDriver, parse_codex_jsonl_events +from evals.drivers.cli.opencode import OpencodeCliDriver, write_opencode_mcp_config from evals.tool_names import ( split_plane_and_client_calls, ) @@ -380,7 +379,7 @@ def test_parse_behaviours(case, tmp_path): def _find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): - from evals import drivers as drivers_mod + from evals.drivers.cli import codex as codex_mod sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" sessions.mkdir(parents=True) @@ -399,18 +398,18 @@ def _find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): ) monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout(tid) + found = codex_mod.find_codex_rollout(tid) assert found is not None assert tid in found.name # Must not return the other concurrent session assert "other-session" not in found.name - assert drivers_mod.find_codex_rollout("does-not-exist-anywhere") is None - assert drivers_mod.find_codex_rollout(None) is None + assert codex_mod.find_codex_rollout("does-not-exist-anywhere") is None + assert codex_mod.find_codex_rollout(None) is None def _find_codex_rollout_session_meta_id(tmp_path, monkeypatch): - from evals import drivers as drivers_mod + from evals.drivers.cli import codex as codex_mod sessions = tmp_path / ".codex" / "sessions" sessions.mkdir(parents=True) @@ -420,7 +419,7 @@ def _find_codex_rollout_session_meta_id(tmp_path, monkeypatch): encoding="utf-8", ) monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - found = drivers_mod.find_codex_rollout("sess-meta-42") + found = codex_mod.find_codex_rollout("sess-meta-42") assert found is not None assert found.name == "rollout-meta-only.jsonl" diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 9a62204a..716dce46 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -14,9 +14,7 @@ from plane.errors.errors import HttpError from evals import cli as run_mod -from evals.drivers import ( - ClaudeCliDriver, -) +from evals.drivers.cli.claude import ClaudeCliDriver from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.report import load_rows, summarize from evals.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult diff --git a/tests/evals/test_package_boundaries.py b/tests/evals/test_package_boundaries.py index b8ec3621..388b6976 100644 --- a/tests/evals/test_package_boundaries.py +++ b/tests/evals/test_package_boundaries.py @@ -50,3 +50,39 @@ def test_import_does_not_cross_layer(module: str, forbidden: tuple[str, ...]): def test_the_probe_can_actually_observe_a_violation(): """A boundary test that cannot fail is worse than none: prove the probe sees imports.""" assert "runner" in loaded_subpackages("evals.runner.live") + + +# The two driver surfaces are independent: the API driver owns its loop and speaks to a +# provider, while a CLI driver supervises a subprocess and reads a recording proxy. Neither +# needs anything the other has. Depth-1 names cannot express this — both live under +# ``drivers`` — so these cases match module prefixes instead. +# +# (module to import, module prefixes it must not drag in) +SURFACE_BOUNDARIES = [ + # Reading the registry must load no surface at all, or get_driver's per-vendor imports + # are decoration: a flat re-export wall here once made every consumer load all five + # agent CLIs, because Python runs a package's __init__ before any submodule. + ("evals.drivers", ("evals.drivers.api.", "evals.drivers.cli.")), + ("evals.drivers.api.base", ("evals.drivers.cli.",)), + ("evals.drivers.api.driver", ("evals.drivers.cli.",)), + ("evals.drivers.cli.base", ("evals.drivers.api.",)), + ("evals.drivers.cli.claude", ("evals.drivers.api.",)), +] + + +def loaded_modules(module: str) -> set[str]: + """Return the full names of the ``evals.*`` modules present after importing ``module``.""" + code = f"import {module}, sys\nprint(' '.join(sorted(m for m in sys.modules if m.startswith('evals'))))" + completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + return set(completed.stdout.split()) + + +@pytest.mark.parametrize(("module", "forbidden"), SURFACE_BOUNDARIES, ids=[case[0] for case in SURFACE_BOUNDARIES]) +def test_import_does_not_cross_driver_surface(module: str, forbidden: tuple[str, ...]): + leaked = sorted(name for name in loaded_modules(module) if name.startswith(forbidden)) + assert not leaked, f"importing {module} loaded {leaked}, which it must not depend on" + + +def test_the_surface_probe_can_actually_observe_a_violation(): + """Same guard as above, for the prefix probe: prove it sees a real intra-surface import.""" + assert "evals.drivers.cli.base" in loaded_modules("evals.drivers.cli.claude") diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 9c056a5f..ba458744 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -14,7 +14,7 @@ import pytest -from evals.drivers import ( +from evals.drivers.cli.sidecar import ( apply_proxy_sidecar, ensure_proxy_pythonpath, load_proxy_sidecar, diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py index dd8c8828..a57a0830 100644 --- a/tests/evals/test_results.py +++ b/tests/evals/test_results.py @@ -10,9 +10,6 @@ import pytest -from evals.drivers import ( - ApiDriver, -) from evals.drivers.api import ( StopReason, ToolCall, @@ -20,6 +17,7 @@ ToolSpec, Turn, ) +from evals.drivers.api.driver import ApiDriver from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.results import ( AGENT_RESULT_COPY_FIELDS, From fdd971414140086abfdbb38370fa194c50145f72 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 18 Aug 2026 20:46:57 +0530 Subject: [PATCH 65/93] Give the shared floor of the package a name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals/ had eighteen top-level modules, and a flat listing said nothing about what they were. Ten of them import nothing else inside evals: they are the vocabulary every other layer sits on, and errors.py at 21 lines looked like a peer of proxy.py at 860. Those ten plus results.py now live in evals/core/. The set is not a judgement call — it is exactly the modules that already satisfied "imports only each other", verified before the move. What makes this a boundary rather than a label is that membership is checked. A package named after its position in the dependency graph becomes a dumping ground unless the position is testable; here it is, so a boundary case asserts that importing any core module loads no evals module outside core. Membership is discovered from the directory rather than listed, because a hand-kept list makes the check opt-in — a module dropped into core/ and omitted from the list could import whatever it liked. Verified by planting a violating module and watching the case name it without any test edit. core/__init__.py deliberately re-exports nothing. A re-export wall there would make importing one core module load all eleven, which is the coupling just removed from drivers/__init__.py. Six modules stay at the top level. cli, cleanup and listing are entry points at the top of the graph, not vocabulary. proxy is a separate program spawned as `python -m evals.proxy`, so that path is public. skip_taxonomy imports the task catalog and result_lifecycle imports skip_taxonomy, so neither is floor — splitting result_lifecycle from results reads oddly but is honest. One assertion had to move rather than survive: BOUNDARIES pinned errors.py as depending on nothing of ours, but it matches depth-1 package names, and once results moved under core its depth-1 name became core, so the entry forbidding results could no longer match. Re-asserted at the granularity that survives the move: errors may load nothing beyond evals and evals.core themselves. --- evals/DESIGN.md | 15 ++++-- evals/core/__init__.py | 8 ++++ evals/{ => core}/changelog.py | 0 evals/{ => core}/errors.py | 0 evals/{ => core}/evidence.py | 0 evals/{ => core}/fixtures.py | 0 evals/{ => core}/results.py | 4 +- evals/{ => core}/server_env.py | 0 evals/{ => core}/state_oracle.py | 0 evals/{ => core}/task_metadata.py | 0 evals/{ => core}/token_counting.py | 0 evals/{ => core}/tool_manifest.py | 0 evals/{ => core}/tool_names.py | 0 evals/drivers/api/base.py | 2 +- evals/drivers/api/driver.py | 24 +++++----- evals/drivers/cli/base.py | 18 ++++---- evals/drivers/cli/claude.py | 2 +- evals/drivers/cli/codex.py | 2 +- evals/drivers/cli/sidecar.py | 2 +- evals/listing.py | 2 +- evals/proxy.py | 4 +- evals/report/command.py | 2 +- evals/report/load.py | 2 +- evals/report/off_surface.py | 6 +-- evals/report/schema_friction.py | 2 +- evals/report/summary.py | 4 +- evals/report/table.py | 4 +- evals/result_lifecycle.py | 2 +- evals/runner/canary.py | 2 +- evals/runner/live.py | 10 ++-- evals/runner/meta.py | 4 +- evals/runner/resume.py | 2 +- evals/seed/build.py | 2 +- evals/seed/customers.py | 2 +- evals/seed/cycles.py | 4 +- evals/seed/gates.py | 2 +- evals/seed/intake.py | 2 +- evals/seed/modules.py | 2 +- evals/seed/projects.py | 2 +- evals/seed/releases.py | 6 +-- evals/seed/states.py | 4 +- evals/seed/work_items.py | 10 ++-- evals/tasks/__init__.py | 2 +- evals/tasks/answers.py | 2 +- evals/tasks/cross.py | 4 +- evals/tasks/debias.py | 4 +- evals/tasks/read.py | 4 +- evals/tasks/schema.py | 4 +- evals/tasks/skip.py | 6 +-- evals/tasks/write.py | 2 +- tests/evals/drivers/test_api_driver.py | 6 +-- tests/evals/drivers/test_cli_driver.py | 2 +- tests/evals/drivers/test_vendors.py | 6 +-- tests/evals/report/test_off_surface.py | 4 +- tests/evals/report/test_summary.py | 2 +- tests/evals/runner/test_live.py | 6 +-- tests/evals/runner/test_resume.py | 2 +- tests/evals/seed/test_gate_tolerance.py | 2 +- tests/evals/seed/test_read_randomization.py | 2 +- tests/evals/seed/test_seed.py | 4 +- tests/evals/tasks/test_debias_verifiers.py | 2 +- tests/evals/tasks/test_output_contracts.py | 2 +- .../evals/tasks/test_verifier_read_errors.py | 4 +- tests/evals/tasks/test_verifiers.py | 2 +- tests/evals/test_evidence.py | 2 +- tests/evals/test_import_compat.py | 4 +- tests/evals/test_package_boundaries.py | 46 ++++++++++++++++++- tests/evals/test_proxy.py | 14 +++--- tests/evals/test_results.py | 24 +++++----- tests/evals/test_token_counting.py | 4 +- tests/evals/test_tool_manifest.py | 2 +- tests/evals/test_tool_names.py | 2 +- 72 files changed, 195 insertions(+), 136 deletions(-) create mode 100644 evals/core/__init__.py rename evals/{ => core}/changelog.py (100%) rename evals/{ => core}/errors.py (100%) rename evals/{ => core}/evidence.py (100%) rename evals/{ => core}/fixtures.py (100%) rename evals/{ => core}/results.py (99%) rename evals/{ => core}/server_env.py (100%) rename evals/{ => core}/state_oracle.py (100%) rename evals/{ => core}/task_metadata.py (100%) rename evals/{ => core}/token_counting.py (100%) rename evals/{ => core}/tool_manifest.py (100%) rename evals/{ => core}/tool_names.py (100%) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 80da24e7..68d2c371 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -342,9 +342,18 @@ evals/ codex.py Codex CLI driver antigravity.py Antigravity CLI driver opencode.py OpenCode CLI driver - results.py run/task result types and common row mapping - tool_names.py whose MCP tool a call is, and what to call it - token_counting.py tool-result token sizing + core/ shared floor: may import only core (+ stdlib/third-party) + changelog.py changelog text normalization helpers + errors.py neutral exceptions (TaskSkipped, …) + evidence.py target-binding evidence labels and sentinels + fixtures.py seeded fixture name/title constants + results.py run/task result types and common row mapping + server_env.py stdio MCP server env construction + state_oracle.py Plane state lookups used as verifier truth + task_metadata.py task tags/needs/prompt persisted in the run's meta header + token_counting.py tool-result token sizing + tool_manifest.py tools/list capture and fingerprinting + tool_names.py whose MCP tool a call is, and what to call it proxy.py stdlib-only JSON-RPC recording relay seed/ Plane fixture creation and teardown report/ summaries, A/B comparison, and multi-surface tables diff --git a/evals/core/__init__.py b/evals/core/__init__.py new file mode 100644 index 00000000..9d696116 --- /dev/null +++ b/evals/core/__init__.py @@ -0,0 +1,8 @@ +"""Shared floor of the evals package. + +Modules here may import only other ``evals.core`` modules (plus the +standard library and third-party packages). They must not import +runners, drivers, seeders, tasks, report code, or the recording proxy. +Callers import ``evals.core.`` directly — this package does not +re-export its submodules. +""" diff --git a/evals/changelog.py b/evals/core/changelog.py similarity index 100% rename from evals/changelog.py rename to evals/core/changelog.py diff --git a/evals/errors.py b/evals/core/errors.py similarity index 100% rename from evals/errors.py rename to evals/core/errors.py diff --git a/evals/evidence.py b/evals/core/evidence.py similarity index 100% rename from evals/evidence.py rename to evals/core/evidence.py diff --git a/evals/fixtures.py b/evals/core/fixtures.py similarity index 100% rename from evals/fixtures.py rename to evals/core/fixtures.py diff --git a/evals/results.py b/evals/core/results.py similarity index 99% rename from evals/results.py rename to evals/core/results.py index 1d4fe44d..163610fb 100644 --- a/evals/results.py +++ b/evals/core/results.py @@ -6,12 +6,12 @@ from dataclasses import dataclass, field from typing import Any, Literal -from evals.token_counting import ( +from evals.core.token_counting import ( TOKEN_ESTIMATE_METHOD, count_result_text_tokens, estimate_result_tokens, ) -from evals.tool_names import split_plane_and_client_calls +from evals.core.tool_names import split_plane_and_client_calls RESULT_SCHEMA_VERSION = 6 TRACE_INTEGRITY_SCHEMA_VERSION = 5 diff --git a/evals/server_env.py b/evals/core/server_env.py similarity index 100% rename from evals/server_env.py rename to evals/core/server_env.py diff --git a/evals/state_oracle.py b/evals/core/state_oracle.py similarity index 100% rename from evals/state_oracle.py rename to evals/core/state_oracle.py diff --git a/evals/task_metadata.py b/evals/core/task_metadata.py similarity index 100% rename from evals/task_metadata.py rename to evals/core/task_metadata.py diff --git a/evals/token_counting.py b/evals/core/token_counting.py similarity index 100% rename from evals/token_counting.py rename to evals/core/token_counting.py diff --git a/evals/tool_manifest.py b/evals/core/tool_manifest.py similarity index 100% rename from evals/tool_manifest.py rename to evals/core/tool_manifest.py diff --git a/evals/tool_names.py b/evals/core/tool_names.py similarity index 100% rename from evals/tool_names.py rename to evals/core/tool_names.py diff --git a/evals/drivers/api/base.py b/evals/drivers/api/base.py index 25731e8c..d512a4f3 100644 --- a/evals/drivers/api/base.py +++ b/evals/drivers/api/base.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Protocol -from evals.results import Usage +from evals.core.results import Usage MODEL_TIERS = frozenset({"standard", "fast"}) diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index dff1c2be..1c3b64fe 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -23,15 +23,7 @@ from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client -from evals.drivers.api.base import ( - KNOWN_API_PROVIDERS, - ModelBackend, - StopReason, - ToolResult, - ToolSpec, - create_backend, -) -from evals.evidence import ( +from evals.core.evidence import ( configured_evidence_labels, normalize_evidence_aggregates, normalize_evidence_sentinels, @@ -40,9 +32,17 @@ observed_aggregates, observed_sentinel_labels, ) -from evals.results import AgentRun, Usage -from evals.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens -from evals.tool_manifest import ToolManifestCapture, tools_page +from evals.core.results import AgentRun, Usage +from evals.core.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens +from evals.core.tool_manifest import ToolManifestCapture, tools_page +from evals.drivers.api.base import ( + KNOWN_API_PROVIDERS, + ModelBackend, + StopReason, + ToolResult, + ToolSpec, + create_backend, +) DEFAULT_MAX_TOKENS = 8192 diff --git a/evals/drivers/cli/base.py b/evals/drivers/cli/base.py index 5d4c9776..fed7c720 100644 --- a/evals/drivers/cli/base.py +++ b/evals/drivers/cli/base.py @@ -22,6 +22,15 @@ from typing import Any from evals import REPO_ROOT +from evals.core.evidence import ( + configured_evidence_labels, + normalize_evidence_aggregates, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_aggregate_labels, + write_evidence_config, +) +from evals.core.results import AgentRun from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess from evals.drivers.cli.sidecar import ( ProxySidecarResult, @@ -31,15 +40,6 @@ load_proxy_sidecar, proxy_wrap_server_command, ) -from evals.evidence import ( - configured_evidence_labels, - normalize_evidence_aggregates, - normalize_evidence_sentinels, - normalize_evidence_targets, - observed_aggregate_labels, - write_evidence_config, -) -from evals.results import AgentRun @dataclass diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index f1d9927d..6632e8d7 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -23,8 +23,8 @@ from pathlib import Path from typing import Any +from evals.core.tool_names import normalize_tool_call, split_plane_and_client_calls from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError -from evals.tool_names import normalize_tool_call, split_plane_and_client_calls def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py index ca4d63c8..5cafdd05 100644 --- a/evals/drivers/cli/codex.py +++ b/evals/drivers/cli/codex.py @@ -16,9 +16,9 @@ from pathlib import Path from typing import Any +from evals.core.tool_names import normalize_tool_call, split_plane_and_client_calls from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput from evals.drivers.cli.process import run_cli_subprocess -from evals.tool_names import normalize_tool_call, split_plane_and_client_calls def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 00632f51..8935af00 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -12,7 +12,7 @@ from typing import Any, Literal from evals import REPO_ROOT -from evals.results import TraceIntegrityReason +from evals.core.results import TraceIntegrityReason ProxyMetaWaitOutcome = Literal["meta_present", "proxy_exited", "proxy_not_observed", "timeout"] diff --git a/evals/listing.py b/evals/listing.py index 2f50c600..fa1d2496 100644 --- a/evals/listing.py +++ b/evals/listing.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from typing import Any -from evals.server_env import stdio_server_env +from evals.core.server_env import stdio_server_env @dataclass diff --git a/evals/proxy.py b/evals/proxy.py index 4cf4a6d2..e1d618a7 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any -from evals.evidence import ( +from evals.core.evidence import ( EVIDENCE_SENTINELS_ENV, consume_evidence_config, fingerprint_evidence_sentinels, @@ -32,7 +32,7 @@ observed_aggregates, observed_fingerprint_labels, ) -from evals.tool_manifest import ToolManifestCapture +from evals.core.tool_manifest import ToolManifestCapture # Single post-EOF / child-exit deadline for the whole shutdown sequence. SHUTDOWN_DEADLINE_S = 10.0 diff --git a/evals/report/command.py b/evals/report/command.py index 404d1a78..e54aee58 100644 --- a/evals/report/command.py +++ b/evals/report/command.py @@ -6,7 +6,7 @@ import sys from pathlib import Path -from evals.results import TaskResult +from evals.core.results import TaskResult from .compare import ab_compare, print_ab_report from .identity import ( diff --git a/evals/report/load.py b/evals/report/load.py index 52c06c2c..6aaf4268 100644 --- a/evals/report/load.py +++ b/evals/report/load.py @@ -9,8 +9,8 @@ from pathlib import Path from typing import Any, Literal +from evals.core.results import TaskResult from evals.result_lifecycle import is_terminal_result -from evals.results import TaskResult DedupeMode = Literal["latest", "none"] ResultRow = TaskResult | dict[str, Any] diff --git a/evals/report/off_surface.py b/evals/report/off_surface.py index 99cc50c8..eebc0d7e 100644 --- a/evals/report/off_surface.py +++ b/evals/report/off_surface.py @@ -7,9 +7,9 @@ from dataclasses import dataclass from typing import Any -from evals.evidence import TARGET_ENTITY_EVIDENCE -from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, CallRecord, TaskResult -from evals.task_metadata import task_metadata_from_rows +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import TRACE_INTEGRITY_SCHEMA_VERSION, CallRecord, TaskResult +from evals.core.task_metadata import task_metadata_from_rows from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import percentile diff --git a/evals/report/schema_friction.py b/evals/report/schema_friction.py index edaf8812..b6a2302f 100644 --- a/evals/report/schema_friction.py +++ b/evals/report/schema_friction.py @@ -5,7 +5,7 @@ from collections import defaultdict from dataclasses import dataclass -from evals.results import TaskResult +from evals.core.results import TaskResult from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import median diff --git a/evals/report/summary.py b/evals/report/summary.py index 600c2c1d..e95b09a7 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -6,9 +6,9 @@ from dataclasses import dataclass from typing import Literal -from evals.results import TRACE_INTEGRITY_SCHEMA_VERSION, TaskResult +from evals.core.results import TRACE_INTEGRITY_SCHEMA_VERSION, TaskResult +from evals.core.task_metadata import TaskMetadata, entry_needs, task_metadata_from_rows from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family -from evals.task_metadata import TaskMetadata, entry_needs, task_metadata_from_rows from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import OffSurfaceMeasurement, measure_off_surface diff --git a/evals/report/table.py b/evals/report/table.py index 453d130d..2f11c417 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any -from evals.results import TaskResult -from evals.task_metadata import TaskMetadata, entry_prompt, task_metadata_from_rows +from evals.core.results import TaskResult +from evals.core.task_metadata import TaskMetadata, entry_prompt, task_metadata_from_rows from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import off_surface_statement diff --git a/evals/result_lifecycle.py b/evals/result_lifecycle.py index ae072028..1ff672b3 100644 --- a/evals/result_lifecycle.py +++ b/evals/result_lifecycle.py @@ -4,7 +4,7 @@ from typing import Any -from evals.results import TaskResult +from evals.core.results import TaskResult from evals.skip_taxonomy import is_expected_environment_capability_skip diff --git a/evals/runner/canary.py b/evals/runner/canary.py index 44fb57cb..93241cc2 100644 --- a/evals/runner/canary.py +++ b/evals/runner/canary.py @@ -6,7 +6,7 @@ import uuid from typing import Any -from evals.errors import TaskSkipped +from evals.core.errors import TaskSkipped from evals.seed import make_plane_client, seed, teardown from evals.tasks.catalog import battery_fingerprint diff --git a/evals/runner/live.py b/evals/runner/live.py index 7cdf854e..894f8258 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -11,19 +11,19 @@ from pathlib import Path from typing import Any +from evals.core.errors import TaskSkipped +from evals.core.evidence import configured_evidence_labels +from evals.core.results import TaskResult, agent_run_to_task_result +from evals.core.server_env import stdio_server_env +from evals.core.task_metadata import build_task_metadata from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS -from evals.errors import TaskSkipped -from evals.evidence import configured_evidence_labels from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys from evals.report.off_surface import off_surface_statement from evals.report.schema_friction import schema_friction_statement from evals.report.summary import completeness_statement, execution_coverage_statement, summarize -from evals.results import TaskResult, agent_run_to_task_result from evals.seed import make_plane_client, seed, teardown from evals.seed.identities import capture_seed_artifacts -from evals.server_env import stdio_server_env -from evals.task_metadata import build_task_metadata from evals.tasks.catalog import battery_fingerprint, task_author, task_fingerprint from evals.tasks.prompts import PromptBindError, format_task_prompt diff --git a/evals/runner/meta.py b/evals/runner/meta.py index 35366c6f..56cab818 100644 --- a/evals/runner/meta.py +++ b/evals/runner/meta.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any -from evals.results import RESULT_SCHEMA_VERSION -from evals.task_metadata import METADATA_FIELD, normalize_task_metadata +from evals.core.results import RESULT_SCHEMA_VERSION +from evals.core.task_metadata import METADATA_FIELD, normalize_task_metadata def read_git_revision() -> str: diff --git a/evals/runner/resume.py b/evals/runner/resume.py index 368a5d71..759a95bb 100644 --- a/evals/runner/resume.py +++ b/evals/runner/resume.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any +from evals.core.results import TaskResult from evals.result_lifecycle import is_terminal_result -from evals.results import TaskResult from .meta import is_meta_or_non_task_row diff --git a/evals/seed/build.py b/evals/seed/build.py index a695074d..a2e6ee95 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -7,7 +7,7 @@ from plane import PlaneClient -from evals.errors import TaskSkipped +from evals.core.errors import TaskSkipped from .customers import ( CUSTOMER_NAME, diff --git a/evals/seed/customers.py b/evals/seed/customers.py index 35057215..18f61118 100644 --- a/evals/seed/customers.py +++ b/evals/seed/customers.py @@ -7,7 +7,7 @@ from plane import PlaneClient from plane.models.customers import CreateCustomer, CreateCustomerRequest -from evals.fixtures import ( +from evals.core.fixtures import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME, diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py index aff2e7d3..d29313ba 100644 --- a/evals/seed/cycles.py +++ b/evals/seed/cycles.py @@ -9,8 +9,8 @@ from plane.models.cycles import CreateCycle, UpdateCycle from plane.models.work_items import UpdateWorkItem -from evals.evidence import set_target_evidence -from evals.fixtures import CYCLE_CURRENT, CYCLE_PAST, PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES +from evals.core.evidence import set_target_evidence +from evals.core.fixtures import CYCLE_CURRENT, CYCLE_PAST, PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES from .identities import record_seeded_entity from .randomize import random_truth_rng, record_randomized_truth diff --git a/evals/seed/gates.py b/evals/seed/gates.py index 35c21d23..872ca9d8 100644 --- a/evals/seed/gates.py +++ b/evals/seed/gates.py @@ -13,7 +13,7 @@ from plane.errors.errors import HttpError -from evals.errors import TaskSkipped +from evals.core.errors import TaskSkipped # Wording a refusal uses when the workspace's plan is what stands in the way. A feature # switched off for a project says "not enabled for this project" instead, which is a diff --git a/evals/seed/intake.py b/evals/seed/intake.py index 66d23c0c..1c35effa 100644 --- a/evals/seed/intake.py +++ b/evals/seed/intake.py @@ -7,7 +7,7 @@ from plane import PlaneClient from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest -from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE from .identities import record_seeded_entity diff --git a/evals/seed/modules.py b/evals/seed/modules.py index 050f7e07..1d1d3112 100644 --- a/evals/seed/modules.py +++ b/evals/seed/modules.py @@ -8,7 +8,7 @@ from plane.models.modules import CreateModule from plane.models.work_items import CreateWorkItem, UpdateWorkItem -from evals.fixtures import MODULE_COMPLETED_TITLES, MODULE_NAME +from evals.core.fixtures import MODULE_COMPLETED_TITLES, MODULE_NAME from .identities import record_seeded_entity from .work_items import find_completed_state, list_states diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 229df2a9..90b8d722 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -11,7 +11,7 @@ from plane.models.work_items import CreateWorkItem from plane.models.workspaces import WorkspaceFeature -from evals.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence +from evals.core.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence from .gates import is_plan_gate from .identities import record_seeded_entity diff --git a/evals/seed/releases.py b/evals/seed/releases.py index 138d9f90..f82a910e 100644 --- a/evals/seed/releases.py +++ b/evals/seed/releases.py @@ -7,9 +7,9 @@ from plane import PlaneClient from plane.models.releases import CreateRelease, UpdateReleaseChangelog -from evals.changelog import changelog_items, normalize_changelog_text -from evals.evidence import set_target_evidence -from evals.fixtures import ( +from evals.core.changelog import changelog_items, normalize_changelog_text +from evals.core.evidence import set_target_evidence +from evals.core.fixtures import ( EVALUATION_RELEASE_TAG_VERSION, RELEASE_CHANGELOG_TEXT, RELEASE_NAME, diff --git a/evals/seed/states.py b/evals/seed/states.py index c840e9d4..0f1d78f4 100644 --- a/evals/seed/states.py +++ b/evals/seed/states.py @@ -7,8 +7,8 @@ from plane import PlaneClient from plane.models.states import CreateState -from evals.evidence import set_target_evidence -from evals.state_oracle import state_name_group_pairs +from evals.core.evidence import set_target_evidence +from evals.core.state_oracle import state_name_group_pairs from .identities import record_seeded_entity from .randomize import random_truth_rng, random_truth_token, record_randomized_truth diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py index a616166d..051be838 100644 --- a/evals/seed/work_items.py +++ b/evals/seed/work_items.py @@ -11,10 +11,10 @@ from plane.models.states import CreateState from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem -from evals.changelog import normalize_changelog_text -from evals.errors import TaskSkipped -from evals.evidence import set_target_count_evidence, set_target_evidence -from evals.fixtures import ( +from evals.core.changelog import normalize_changelog_text +from evals.core.errors import TaskSkipped +from evals.core.evidence import set_target_count_evidence, set_target_evidence +from evals.core.fixtures import ( BLOCKING_REFERENCE_ADDRESS, BLOCKING_SOURCE_TITLE, BLOCKING_TARGET_TITLE, @@ -27,7 +27,7 @@ UNFINISHED_CYCLE_TITLES, WORK_ITEM_FIXTURES, ) -from evals.state_oracle import worklog_summary_item_ids +from evals.core.state_oracle import worklog_summary_item_ids from .gates import plan_gate_skips from .identities import record_seeded_entity diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py index ac57f436..f7d93c0f 100644 --- a/evals/tasks/__init__.py +++ b/evals/tasks/__init__.py @@ -1,6 +1,6 @@ """Public task catalog and verifier API.""" -from evals.errors import TaskSkipped +from evals.core.errors import TaskSkipped from evals.tasks.answers import ( contract_values, get_final_text, diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py index 00dd50ed..cc5454b8 100644 --- a/evals/tasks/answers.py +++ b/evals/tasks/answers.py @@ -7,7 +7,7 @@ from html import unescape from typing import Any -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE def word_boundary(value: str) -> re.Pattern[str]: diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py index 8f59b7f9..8484a1a2 100644 --- a/evals/tasks/cross.py +++ b/evals/tasks/cross.py @@ -4,8 +4,8 @@ from typing import Any -from evals.changelog import changelog_items, normalize_changelog_text -from evals.fixtures import ( +from evals.core.changelog import changelog_items, normalize_changelog_text +from evals.core.fixtures import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, R1_TITLE, diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py index 93d3ff1b..f8e7d5ae 100644 --- a/evals/tasks/debias.py +++ b/evals/tasks/debias.py @@ -6,7 +6,7 @@ from plane.models.query_params import RetrieveQueryParams -from evals.fixtures import ( +from evals.core.fixtures import ( CUSTOMER_NAME, CYCLE_CURRENT, DEBIAS_CUSTOMER_PROP_DISPLAY, @@ -17,7 +17,7 @@ W3_TITLE, W8_TITLE, ) -from evals.state_oracle import worklog_summary_item_ids +from evals.core.state_oracle import worklog_summary_item_ids from evals.tasks.answers import ( answer_with_provenance, contract_values, diff --git a/evals/tasks/read.py b/evals/tasks/read.py index 03ae80fb..8e5c0223 100644 --- a/evals/tasks/read.py +++ b/evals/tasks/read.py @@ -5,8 +5,8 @@ from collections import Counter from typing import Any -from evals.fixtures import R1_TITLE, R5_TITLE -from evals.state_oracle import state_name_group_pairs +from evals.core.fixtures import R1_TITLE, R5_TITLE +from evals.core.state_oracle import state_name_group_pairs from evals.tasks.answers import ( answer_with_provenance, contract_values, diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py index 31d6bb54..f5f9fa5b 100644 --- a/evals/tasks/schema.py +++ b/evals/tasks/schema.py @@ -7,8 +7,8 @@ from plane.errors.errors import HttpError from plane.models.enums import PropertyType -from evals.errors import TaskSkipped -from evals.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE +from evals.core.errors import TaskSkipped +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE from evals.tasks.lookups import as_id, find_item_by_name from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error diff --git a/evals/tasks/skip.py b/evals/tasks/skip.py index 0822efb5..19b9abda 100644 --- a/evals/tasks/skip.py +++ b/evals/tasks/skip.py @@ -1,11 +1,11 @@ -"""Retired import path for :class:`evals.errors.TaskSkipped`. +"""Retired import path for :class:`evals.core.errors.TaskSkipped`. Kept because it shipped and ``tests/evals/test_import_compat.py`` pins it. Nothing in the -package imports it: the canonical home is ``evals.errors``, which depends on nothing. Every +package imports it: the canonical home is ``evals.core.errors``, which depends on nothing. Every source module routed through here for a while, which left the neutral module unused and made a compat shim look like the real one. """ -from evals.errors import TaskSkipped as TaskSkipped +from evals.core.errors import TaskSkipped as TaskSkipped __all__ = ["TaskSkipped"] diff --git a/evals/tasks/write.py b/evals/tasks/write.py index fc37c98c..6a932321 100644 --- a/evals/tasks/write.py +++ b/evals/tasks/write.py @@ -7,7 +7,7 @@ from plane.errors.errors import HttpError from plane.models.query_params import PaginatedQueryParams, RetrieveQueryParams, WorkItemQueryParams -from evals.fixtures import ( +from evals.core.fixtures import ( CYCLE_CURRENT, CYCLE_PAST, MODULE_COMPLETED_TITLES, diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 3bd50fac..7e236726 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -10,6 +10,9 @@ import pytest +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.token_counting import estimate_result_tokens +from evals.core.tool_manifest import tool_manifest_fingerprint from evals.drivers.api import ( KNOWN_API_PROVIDERS, AnthropicBackend, @@ -26,9 +29,6 @@ unregister_backend, ) from evals.drivers.api.driver import ApiDriver -from evals.evidence import TARGET_ENTITY_EVIDENCE -from evals.token_counting import estimate_result_tokens -from evals.tool_manifest import tool_manifest_fingerprint from tests.evals.conftest import case_params diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index 6fc8a051..bfa317c3 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -14,6 +14,7 @@ import pytest import tomllib +from evals.core.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from evals.drivers.cli.antigravity import AntigravityCliDriver from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError from evals.drivers.cli.claude import ClaudeCliDriver @@ -30,7 +31,6 @@ proxy_wrap_server_command, wait_for_proxy_meta, ) -from evals.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE from tests.evals.conftest import case_params diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index e4838492..a8013704 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -10,6 +10,9 @@ import pytest +from evals.core.tool_names import ( + split_plane_and_client_calls, +) from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api.driver import ApiDriver from evals.drivers.cli.antigravity import ( @@ -26,9 +29,6 @@ ) from evals.drivers.cli.codex import CodexCliDriver, parse_codex_jsonl_events from evals.drivers.cli.opencode import OpencodeCliDriver, write_opencode_mcp_config -from evals.tool_names import ( - split_plane_and_client_calls, -) from tests.evals.conftest import case_params CLAUDE_JSON_RESULT = { diff --git a/tests/evals/report/test_off_surface.py b/tests/evals/report/test_off_surface.py index e4060a62..bc589506 100644 --- a/tests/evals/report/test_off_surface.py +++ b/tests/evals/report/test_off_surface.py @@ -5,7 +5,8 @@ from pathlib import Path from typing import Any -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import RESULT_SCHEMA_VERSION from evals.report import ( ANSWER_WITHOUT_PROVENANCE, IMPLAUSIBLY_FEW_CALLS, @@ -17,7 +18,6 @@ print_table, summarize, ) -from evals.results import RESULT_SCHEMA_VERSION def _row( diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 55d9e669..d22cb08f 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -8,6 +8,7 @@ import pytest from evals import report as report_mod +from evals.core.results import RESULT_SCHEMA_VERSION from evals.report import ( completeness_statement, execution_coverage_statement, @@ -18,7 +19,6 @@ summarize, wilson_interval, ) -from evals.results import RESULT_SCHEMA_VERSION from tests.evals.conftest import case_params diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index 716dce46..cd58a1c7 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -14,10 +14,10 @@ from plane.errors.errors import HttpError from evals import cli as run_mod +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult from evals.drivers.cli.claude import ClaudeCliDriver -from evals.evidence import TARGET_ENTITY_EVIDENCE from evals.report import load_rows, summarize -from evals.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult from evals.runner import ( is_infra_cli_stop_reason, run_live, @@ -302,7 +302,7 @@ def run_task(self, *args, **kwargs): def _run_timeout_agent_is_infra_cli(tmp_path, monkeypatch, _capsys): - from evals.results import agent_run_to_harness_dict + from evals.core.results import agent_run_to_harness_dict out = tmp_path / "rows.jsonl" fake_plane = MagicMock() diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py index 0e81ae1b..5513c4e0 100644 --- a/tests/evals/runner/test_resume.py +++ b/tests/evals/runner/test_resume.py @@ -9,10 +9,10 @@ import pytest +from evals.core.results import AgentRun from evals.report import ( is_meta_row, ) -from evals.results import AgentRun from evals.runner import ( is_meta_or_non_task_row, load_resume_skip_keys, diff --git a/tests/evals/seed/test_gate_tolerance.py b/tests/evals/seed/test_gate_tolerance.py index b09a1f1a..3e0ec2a3 100644 --- a/tests/evals/seed/test_gate_tolerance.py +++ b/tests/evals/seed/test_gate_tolerance.py @@ -14,7 +14,7 @@ import pytest from plane.errors.errors import HttpError -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import seed_customer, seed_release from evals.tasks.skip import TaskSkipped diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py index 97d427ec..019278a3 100644 --- a/tests/evals/seed/test_read_randomization.py +++ b/tests/evals/seed/test_read_randomization.py @@ -7,7 +7,7 @@ import pytest -from evals.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels +from evals.core.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels from evals.seed.cycles import seed_cycles from evals.seed.states import seed_r7_state_oracle from evals.seed.work_items import require_activities, seed_work_items diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index e5932563..beffb8ce 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -13,8 +13,8 @@ from evals import cleanup as cleanup_mod from evals import seed as seed_mod -from evals.errors import TaskSkipped -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.errors import TaskSkipped +from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import ( R5_TITLE, create_project_with_identifier_retry, diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py index 6d644ddd..53fb86e6 100644 --- a/tests/evals/tasks/test_debias_verifiers.py +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from typing import Any -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import W2_TITLE from evals.tasks.debias import ( I1_TITLE, diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py index 1bead89b..4fd38654 100644 --- a/tests/evals/tasks/test_output_contracts.py +++ b/tests/evals/tasks/test_output_contracts.py @@ -8,7 +8,7 @@ import pytest -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, W2_TITLE, W8_TITLE from evals.tasks.cross import verify_c2 from evals.tasks.debias import verify_i2, verify_l2, verify_l5 diff --git a/tests/evals/tasks/test_verifier_read_errors.py b/tests/evals/tasks/test_verifier_read_errors.py index e151375e..d0abebe8 100644 --- a/tests/evals/tasks/test_verifier_read_errors.py +++ b/tests/evals/tasks/test_verifier_read_errors.py @@ -11,8 +11,8 @@ import pytest from plane.errors.errors import HttpError -from evals.changelog import normalize_changelog_text -from evals.fixtures import ( +from evals.core.changelog import normalize_changelog_text +from evals.core.fixtures import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, CYCLE_CURRENT, diff --git a/tests/evals/tasks/test_verifiers.py b/tests/evals/tasks/test_verifiers.py index 57916fd7..dfc770f1 100644 --- a/tests/evals/tasks/test_verifiers.py +++ b/tests/evals/tasks/test_verifiers.py @@ -10,7 +10,7 @@ import pytest from plane.errors.errors import HttpError -from evals.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, diff --git a/tests/evals/test_evidence.py b/tests/evals/test_evidence.py index ff14fc68..60746636 100644 --- a/tests/evals/test_evidence.py +++ b/tests/evals/test_evidence.py @@ -6,7 +6,7 @@ def test_aggregate_evidence_alone_counts_as_registered_target_bound_evidence(): none. L2 failed that way on every repetition of the first full-catalog battery, after the seeding bug that had hidden it was fixed. """ - from evals.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels + from evals.core.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels targets = {TARGET_ENTITY_EVIDENCE: ("wi-1",)} aggregates = {TARGET_ENTITY_EVIDENCE: ({"kind": "total_count", "value": 3},)} diff --git a/tests/evals/test_import_compat.py b/tests/evals/test_import_compat.py index 253b1059..2862fb60 100644 --- a/tests/evals/test_import_compat.py +++ b/tests/evals/test_import_compat.py @@ -10,8 +10,8 @@ def test_seed_and_task_packages_import_in_either_order_with_legacy_reexports(): root = Path(__file__).parents[2] assertions = """ -from evals.errors import TaskSkipped as NeutralTaskSkipped -from evals.fixtures import CUSTOMER_NAME as NeutralCustomerName +from evals.core.errors import TaskSkipped as NeutralTaskSkipped +from evals.core.fixtures import CUSTOMER_NAME as NeutralCustomerName from evals.seed import CUSTOMER_NAME, R1_TITLE from evals.seed.customers import is_evaluation_customer_name from evals.seed.releases import EVALUATION_RELEASE_TAG_VERSION diff --git a/tests/evals/test_package_boundaries.py b/tests/evals/test_package_boundaries.py index 388b6976..f4d1abae 100644 --- a/tests/evals/test_package_boundaries.py +++ b/tests/evals/test_package_boundaries.py @@ -10,6 +10,7 @@ from __future__ import annotations +import pathlib import subprocess import sys @@ -25,8 +26,6 @@ # A pure token-counting helper once imported the live runner, so importing it loaded # every driver, seeder, task and report module in the tree. ("evals.listing", ("runner", "drivers", "seed", "tasks", "report")), - # The neutral exception module is the floor: it may depend on nothing of ours. - ("evals.errors", ("runner", "drivers", "seed", "tasks", "report", "proxy", "results")), ] @@ -86,3 +85,46 @@ def test_import_does_not_cross_driver_surface(module: str, forbidden: tuple[str, def test_the_surface_probe_can_actually_observe_a_violation(): """Same guard as above, for the prefix probe: prove it sees a real intra-surface import.""" assert "evals.drivers.cli.base" in loaded_modules("evals.drivers.cli.claude") + + +# Shared floor: modules under evals.core may import only each other (plus stdlib / +# third-party). A flat dump of helpers into core would silently reintroduce the +# invisible shared vocabulary this package exists to make visible. +# +# Membership is discovered, not listed. Naming a package after its position in the graph +# only holds if the position is checked, and a hand-maintained list makes that opt-in: a +# module dropped into core/ and left out of the list would import whatever it liked. +CORE_MODULES = tuple( + f"evals.core.{path.stem}" + for path in sorted((pathlib.Path(__file__).parents[2] / "evals" / "core").glob("*.py")) + if path.stem != "__init__" +) + + +def test_core_is_not_empty(): + """Guard the discovery above: a bad glob would make every core case vanish silently.""" + assert len(CORE_MODULES) >= 11, CORE_MODULES + + +@pytest.mark.parametrize("module", CORE_MODULES, ids=list(CORE_MODULES)) +def test_core_imports_only_core(module: str): + """Importing any core module must load no evals module outside evals.core.""" + loaded = loaded_modules(module) + leaked = sorted( + name + for name in loaded + if name.startswith("evals.") and name != "evals.core" and not name.startswith("evals.core.") + ) + assert not leaked, f"importing {module} loaded non-core evals modules: {leaked}" + + +def test_the_exception_module_is_the_floor_of_the_floor(): + """``errors`` may depend on nothing of ours at all, not even its core siblings. + + ``BOUNDARIES`` used to assert this, but it matches depth-1 package names, and once + ``results`` moved under ``core`` its depth-1 name became ``core`` — so the entry + forbidding ``results`` could no longer match anything. Asserted here at the granularity + that survives the move. + """ + siblings = sorted(name for name in loaded_modules("evals.core.errors") if name != "evals.core.errors") + assert siblings == ["evals", "evals.core"], siblings diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index ba458744..72899afd 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -14,6 +14,13 @@ import pytest +from evals.core.evidence import ( + EVIDENCE_SENTINELS_ENV, + TARGET_ENTITY_EVIDENCE, + consume_evidence_config, + write_evidence_config, +) +from evals.core.results import AgentRun, TaskResult, agent_run_to_task_result from evals.drivers.cli.sidecar import ( apply_proxy_sidecar, ensure_proxy_pythonpath, @@ -22,12 +29,6 @@ proxy_pid_path, proxy_session_paths, ) -from evals.evidence import ( - EVIDENCE_SENTINELS_ENV, - TARGET_ENTITY_EVIDENCE, - consume_evidence_config, - write_evidence_config, -) from evals.proxy import ( SHUTDOWN_DEADLINE_S, SidecarRecorder, @@ -39,7 +40,6 @@ ) from evals.proxy import main as proxy_main from evals.report.summary import summarize -from evals.results import AgentRun, TaskResult, agent_run_to_task_result from evals.runner.live import _record_trace_infra from tests.evals.conftest import case_params diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py index a57a0830..33548968 100644 --- a/tests/evals/test_results.py +++ b/tests/evals/test_results.py @@ -10,16 +10,8 @@ import pytest -from evals.drivers.api import ( - StopReason, - ToolCall, - ToolResult, - ToolSpec, - Turn, -) -from evals.drivers.api.driver import ApiDriver -from evals.evidence import TARGET_ENTITY_EVIDENCE -from evals.results import ( +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import ( AGENT_RESULT_COPY_FIELDS, AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS, RESULT_SCHEMA_VERSION, @@ -30,10 +22,18 @@ Usage, agent_run_to_harness_dict, ) -from evals.token_counting import estimate_result_tokens -from evals.tool_names import ( +from evals.core.token_counting import estimate_result_tokens +from evals.core.tool_names import ( normalize_tool_call, ) +from evals.drivers.api import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, +) +from evals.drivers.api.driver import ApiDriver from tests.evals.conftest import case_params diff --git a/tests/evals/test_token_counting.py b/tests/evals/test_token_counting.py index fca9b34a..f479f114 100644 --- a/tests/evals/test_token_counting.py +++ b/tests/evals/test_token_counting.py @@ -6,8 +6,8 @@ import pytest -from evals.results import AgentRun, agent_run_to_harness_dict -from evals.token_counting import estimate_result_tokens +from evals.core.results import AgentRun, agent_run_to_harness_dict +from evals.core.token_counting import estimate_result_tokens @pytest.mark.parametrize("has_tokenizer", [True, False], ids=["importable-tokenizer", "estimator-fallback"]) diff --git a/tests/evals/test_tool_manifest.py b/tests/evals/test_tool_manifest.py index ae33f2ff..35c16032 100644 --- a/tests/evals/test_tool_manifest.py +++ b/tests/evals/test_tool_manifest.py @@ -2,7 +2,7 @@ from __future__ import annotations -from evals.tool_manifest import ToolManifestCapture, tool_manifest_fingerprint +from evals.core.tool_manifest import ToolManifestCapture, tool_manifest_fingerprint def test_same_tool_names_with_different_schemas_have_different_manifest_fingerprints(): diff --git a/tests/evals/test_tool_names.py b/tests/evals/test_tool_names.py index 8fe6068b..3cd33bc3 100644 --- a/tests/evals/test_tool_names.py +++ b/tests/evals/test_tool_names.py @@ -2,7 +2,7 @@ from __future__ import annotations -from evals.tool_names import ( +from evals.core.tool_names import ( is_plane_mcp_tool, strip_mcp_prefix, ) From 637d2a7e5186e1477cf1034f2ad6b559731b6e3b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 18 Aug 2026 21:18:37 +0530 Subject: [PATCH 66/93] Test the five verifiers that could pass a wrong answer W1, W9, S1, S2 and S4 had no behavioural test. Every other part of the harness now has an instrument pointed at it, so these five were the last path from an agent mistake to an inflated headline. The cases ask what a loosened verifier would accept, rather than covering the happy path. W1 matches the auth label by seeded id, so a decoy renamed label fails, and it fails closed when the seed has no label id rather than letting the requirement vanish. W9 needs all three titles at high, not two of three. S1 only accepts a Severity reachable by type-scoped listing and distinguishes three outcomes that must stay distinct: authoritative 404 is a clean failure, a 500 is a VerifierReadError outside the denominator, and an unseeded bug type is a skip. S2 needs both the Fibonacci subset and the item estimate, via either the expanded or the UUID shape. S4 turns on the sign, so swapping accept and decline fails. Ten loosenings were applied to the verifier modules one at a time and all ten were caught, each by the case written for it. --- tests/evals/tasks/test_verifiers.py | 369 +++++++++++++++++++++++++++- 1 file changed, 368 insertions(+), 1 deletion(-) diff --git a/tests/evals/tasks/test_verifiers.py b/tests/evals/tasks/test_verifiers.py index dfc770f1..96843806 100644 --- a/tests/evals/tasks/test_verifiers.py +++ b/tests/evals/tasks/test_verifiers.py @@ -10,7 +10,9 @@ import pytest from plane.errors.errors import HttpError +from evals.core.errors import TaskSkipped from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE from evals.seed import ( CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, @@ -24,16 +26,19 @@ ) from evals.tasks.cross import verify_c1 from evals.tasks.read import verify_r3 -from evals.tasks.schema import verify_s3, verify_s5 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.verification import VerifierReadError from evals.tasks.write import ( W10_PAGE_BODY, W10_PAGE_NAME, + verify_w1, verify_w3, verify_w4, verify_w5, verify_w6, verify_w7, verify_w8, + verify_w9, verify_w10, ) @@ -694,3 +699,365 @@ def test_s5_requires_all_three_features_not_a_majority(cycle_view, tracking, cus assert ok is want, note for text in expect: assert text in note, note + + +# --------------------------------------------------------------------------- +# The five verifiers that shipped without a behavioural test: W1, W9, S1, S2, S4. +# Each case below is one a *wrong* verifier would accept, so a regression that +# loosens a check fails here rather than inflating a battery score. +# --------------------------------------------------------------------------- + +W1_TITLE = "Login page 500s on empty password" +ME_ID = "me-1" + + +class _W1Plane: + def __init__(self, items: list[Any], detail: Any): + self._items = items + self._detail = detail + self.work_items = SimpleNamespace(list=lambda **kw: _Page(self._items), retrieve=self._retrieve) + self.users = SimpleNamespace(get_me=lambda **kw: SimpleNamespace(id=ME_ID)) + + def _retrieve(self, **kw): + # W1 verifies the newest duplicate, so the detail must be keyed by the id it asked for. + return self._detail(kw["work_item_id"]) if callable(self._detail) else self._detail + + +def _w1_detail( + *, priority: str = "urgent", assignees: tuple[str, ...] = (ME_ID,), labels: tuple[str, ...] = ("auth-id",) +): + return SimpleNamespace( + priority=priority, + assignees=[SimpleNamespace(id=a) for a in assignees], + labels=[SimpleNamespace(id=lid) for lid in labels], + ) + + +@pytest.mark.parametrize( + ("items", "detail", "ctx_labels", "want", "expect"), + [ + pytest.param( + [_item("w1", W1_TITLE)], _w1_detail(), {"auth": "auth-id"}, True, "auth label attached", id="all-three-met" + ), + pytest.param([], _w1_detail(), {"auth": "auth-id"}, False, "not found", id="never-created"), + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(priority="high"), + {"auth": "auth-id"}, + False, + "want urgent", + id="priority-close-but-wrong", + ), + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(assignees=("someone-else",)), + {"auth": "auth-id"}, + False, + "missing me", + id="assigned-to-the-wrong-person", + ), + # A label *named* auth is not the seeded label. Matching on name would pass this. + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(labels=("decoy-id",)), + {"auth": "auth-id"}, + False, + "missing auth", + id="decoy-label-with-the-right-name", + ), + # Fail closed: a seed that never produced the label must not make the requirement vanish. + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(), + {}, + False, + "auth label id missing from seed ctx", + id="seed-lost-the-label", + ), + ], +) +def test_w1_requires_all_three_conditions_and_the_seeded_label_id(items, detail, ctx_labels, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": ctx_labels} + ok, note = asyncio.run(verify_w1(_W1Plane(items, detail), ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_w1_verifies_the_newest_duplicate_and_says_so(): + """Two items share the title; only the newest satisfies the ask.""" + items = [ + _item("old", W1_TITLE, created_at="2026-01-01T00:00:00Z"), + _item("new", W1_TITLE, created_at="2026-06-01T00:00:00Z"), + ] + details = {"old": _w1_detail(priority="low"), "new": _w1_detail()} + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"auth": "auth-id"}} + ok, note = asyncio.run(verify_w1(_W1Plane(items, lambda wid: details[wid]), ctx, _run())) + assert ok is True, note + assert "2 items with title" in note, note + + +W9_TITLES = ( + "Checkout times out on 3DS challenge", + "Session cookie not rotated after login", + "Inventory count goes negative under load", +) + + +class _W9Plane: + def __init__(self, priorities: dict[str, str], missing: tuple[str, ...] = ()): + self._priorities = priorities + rows = [_item(f"id-{i}", t) for i, t in enumerate(W9_TITLES) if t not in missing] + self.work_items = SimpleNamespace( + list=lambda **kw: _Page(rows), + retrieve=lambda **kw: SimpleNamespace(priority=self._priorities.get(kw["work_item_id"])), + ) + + +@pytest.mark.parametrize( + ("priorities", "missing", "want", "expect"), + [ + pytest.param( + {"id-0": "high", "id-1": "high", "id-2": "high"}, (), True, "3 items priority=high", id="all-three" + ), + # The majority trap: two of three is a failed task, not a pass. + pytest.param({"id-0": "high", "id-1": "high", "id-2": "medium"}, (), False, "Inventory", id="two-of-three"), + pytest.param({"id-0": "high", "id-1": "high"}, (W9_TITLES[2],), False, "missing", id="one-never-existed"), + pytest.param({"id-0": "High", "id-1": "HIGH", "id-2": "high"}, (), True, "3 items", id="priority-case-varies"), + pytest.param({"id-0": None, "id-1": "high", "id-2": "high"}, (), False, "Checkout", id="priority-unset"), + ], +) +def test_w9_requires_all_three_items_not_a_majority(priorities, missing, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = asyncio.run(verify_w9(_W9Plane(priorities, missing), ctx, _run())) + assert ok is want, note + assert expect in note, note + + +class _S1Props: + """Type-scoped property listing, with the options collection as a separate endpoint.""" + + def __init__(self, props: list[Any] | Exception, options: list[Any] | Exception | None = None): + self._props = props + self._options = options if options is not None else [] + self.list = self._list + self.options = SimpleNamespace(list=self._options_list) + + def _list(self, **kw): + if isinstance(self._props, Exception): + raise self._props + return self._props + + def _options_list(self, **kw): + if isinstance(self._options, Exception): + raise self._options + return self._options + + +def _severity(*, property_type: Any = "OPTION", options: tuple[str, ...] = ("Critical", "Major", "Minor"), **kw: Any): + return SimpleNamespace( + id="sev-1", + display_name="Severity", + property_type=property_type, + options=[SimpleNamespace(name=name) for name in options], + **kw, + ) + + +@pytest.mark.parametrize( + ("props", "options", "want", "expect"), + [ + pytest.param( + [_severity()], None, True, "Severity OPTION with Critical/Major/Minor", id="option-with-all-three" + ), + pytest.param([_severity(property_type="TEXT")], None, False, "want OPTION", id="text-instead-of-option"), + pytest.param( + [_severity(options=("Critical", "Major"))], None, False, "missing ['minor']", id="one-choice-short" + ), + pytest.param( + [_severity(options=("CRITICAL", "major", "MiNoR"))], + None, + True, + "Severity OPTION", + id="choice-casing-varies", + ), + # A Severity bound to some other work item type does not satisfy "on the Bug type". + pytest.param([_severity(issue_type="other-type")], None, False, "not found", id="attached-to-the-wrong-type"), + pytest.param([], None, False, "not found", id="never-created"), + # An empty inline collection falls back to the options endpoint. + pytest.param( + [_severity(options=())], + [SimpleNamespace(name="Critical"), SimpleNamespace(name="Major"), SimpleNamespace(name="Minor")], + True, + "Severity OPTION", + id="options-only-on-the-endpoint", + ), + # A 404 on the options endpoint means the choices definitively are not there. + pytest.param([_severity(options=())], _http404(), False, "missing", id="options-endpoint-404"), + # A type-scoped 404 is authoritative absence, not an infrastructure failure. + pytest.param(_http404(), None, False, "type-scoped list empty/404", id="type-scoped-404-is-a-real-failure"), + ], +) +def test_s1_requires_an_option_severity_attached_to_the_bug_type(props, options, want, expect): + plane = SimpleNamespace(work_item_properties=_S1Props(props, options)) + ctx = {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug-type-1"}} + ok, note = asyncio.run(verify_s1(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_s1_skips_when_the_bug_type_was_never_seeded(): + """No fixture means the question was never asked — not an agent failure.""" + plane = SimpleNamespace(work_item_properties=_S1Props([_severity()])) + with pytest.raises(TaskSkipped): + asyncio.run(verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + + +def test_s1_surfaces_a_non_404_read_failure_as_infrastructure(): + """A 500 while reading authoritative state must not be scored as a failed task.""" + plane = SimpleNamespace(work_item_properties=_S1Props(HttpError("boom", status_code=500, response={}))) + ctx = {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug-type-1"}} + with pytest.raises(VerifierReadError): + asyncio.run(verify_s1(plane, ctx, _run())) + + +class _S2Plane: + def __init__( + self, + *, + values: tuple[str, ...], + estimate_point: Any, + item: bool = True, + retrieve_error: Exception | None = None, + ): + self._error = retrieve_error + self._points = [SimpleNamespace(id=f"pt-{v}", value=v) for v in values] + self._estimate_point = estimate_point + rows = [_item("w8", W8_TITLE)] if item else [] + self.estimates = SimpleNamespace(retrieve=self._retrieve, list_points=lambda **kw: self._points) + self.work_items = SimpleNamespace( + list=lambda **kw: _Page(rows), + retrieve=lambda **kw: SimpleNamespace(estimate_point=self._estimate_point), + ) + + def _retrieve(self, **kw): + if self._error: + raise self._error + return SimpleNamespace(id="est-1") + + +FIB = ("1", "2", "3", "5", "8") + + +@pytest.mark.parametrize( + ("plane", "want", "expect"), + [ + pytest.param( + _S2Plane(values=FIB, estimate_point="pt-5"), True, "item estimate_point=5", id="scale-and-item-both-right" + ), + # Both halves are required; a correct scale with the wrong point is not a pass. + pytest.param(_S2Plane(values=FIB, estimate_point="pt-3"), False, "want 5", id="item-points-at-the-wrong-value"), + pytest.param( + _S2Plane(values=("1", "2", "3", "5"), estimate_point="pt-5"), + False, + "missing fib subset", + id="scale-missing-8", + ), + pytest.param(_S2Plane(values=FIB, estimate_point=None), False, "want 5", id="item-has-no-estimate"), + pytest.param(_S2Plane(values=FIB, estimate_point="pt-5", item=False), False, "missing", id="target-item-gone"), + # estimate_point may arrive expanded rather than as a UUID; both are the same end state. + pytest.param( + _S2Plane(values=FIB, estimate_point=SimpleNamespace(value="5")), + True, + "item estimate value=5", + id="expanded-estimate-point", + ), + # No estimate at all reads as "the requested scale was never created", not an error. + pytest.param( + _S2Plane(values=FIB, estimate_point="pt-5", retrieve_error=_http404()), + False, + "was not created", + id="estimate-404", + ), + ], +) +def test_s2_requires_both_the_fibonacci_scale_and_the_item_estimate(plane, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = asyncio.run(verify_s2(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +class _S4Intake: + def __init__( + self, statuses: dict[str, int | None], *, retrieve_works: bool = True, list_error: Exception | None = None + ): + self._statuses = statuses + self._retrieve_works = retrieve_works + self._list_error = list_error + + def retrieve(self, **kw): + if not self._retrieve_works: + raise _http404() + return SimpleNamespace(status=self._statuses.get(kw["work_item_id"])) + + def list(self, **kw): + if self._list_error: + raise self._list_error + return _Page( + [ + SimpleNamespace( + status=self._statuses.get("billing-1"), issue_detail=SimpleNamespace(name=INTAKE_BILLING_TITLE) + ), + SimpleNamespace( + status=self._statuses.get("spam-1"), issue_detail=SimpleNamespace(name=INTAKE_SPAM_TITLE) + ), + ] + ) + + +def _s4_ctx(*, billing: str | None = "billing-1", spam: str | None = "spam-1"): + return { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": billing}, "spam": {"issue_id": spam}}, + } + + +@pytest.mark.parametrize( + ("statuses", "ctx", "want", "expect"), + [ + pytest.param( + {"billing-1": 1, "spam-1": -1}, _s4_ctx(), True, "billing accepted; spam declined", id="right-call-on-both" + ), + # The sign is the whole task: triaging both the same way is not partial credit. + pytest.param({"billing-1": -1, "spam-1": 1}, _s4_ctx(), False, "billing status=-1", id="decisions-swapped"), + pytest.param({"billing-1": 1, "spam-1": 1}, _s4_ctx(), False, "spam status=1", id="accepted-the-spam-too"), + pytest.param( + {"billing-1": None, "spam-1": -1}, _s4_ctx(), False, "billing status=None", id="billing-untouched" + ), + pytest.param( + {"billing-1": 1, "spam-1": -1}, _s4_ctx(billing=None), False, "billing status=None", id="seed-lost-the-id" + ), + ], +) +def test_s4_requires_the_opposite_decision_on_each_intake_row(statuses, ctx, want, expect): + plane = SimpleNamespace(intake=_S4Intake(statuses)) + ok, note = asyncio.run(verify_s4(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_s4_falls_back_to_the_intake_list_when_retrieve_is_unavailable(): + """Retrieve is optional; the list is independently authoritative for the same rows.""" + plane = SimpleNamespace(intake=_S4Intake({"billing-1": 1, "spam-1": -1}, retrieve_works=False)) + ok, note = asyncio.run(verify_s4(plane, _s4_ctx(), _run())) + assert ok is True, note + + +def test_s4_surfaces_a_double_read_failure_as_infrastructure(): + """With neither endpoint readable, the state is unknown — not declined.""" + plane = SimpleNamespace( + intake=_S4Intake({}, retrieve_works=False, list_error=HttpError("boom", status_code=500, response={})) + ) + with pytest.raises(VerifierReadError): + asyncio.run(verify_s4(plane, _s4_ctx(), _run())) From bd4b7cd61ac35886b1f6cf59a0c01fe1ba2e216b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Wed, 19 Aug 2026 09:10:44 +0530 Subject: [PATCH 67/93] Stop a parked stdin read from discarding a whole trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every claude-cli row was charged to infrastructure. The cause was one opaque boolean: the proxy reported pumps_alive if any of its three pump threads was still running at finalization, and one session with a live pump vetoed the merged trace. Claude Code opens two MCP sessions per task and signals them on exit, so the session that did the work finalizes with its stdin read parked on a client that will never write again. Measured on a live run: the only live stream was stdin, the call was recorded before the signal, and the trace was discarded anyway. A live stdin pump after a signal or child exit cannot have lost anything — no further request can arrive, and an in-flight request that never got its answer is already counted as an unmatched response. A live output pump is different: the server may have been mid-reply when the deadline expired. The proxy now records which streams were pumping rather than a bare boolean, which is what made this diagnosable at all. The raw fact stays in the sidecar; the consumer decides what it means, and sidecars written before the per-stream detail keep the old stricter reading instead of being reinterpreted after the fact. Verified end to end: the R6 smoke that failed as infra_trace now evaluates, with trace_integrity true, and scores as a genuine model failure naming zero Plane calls as the reason. --- evals/drivers/cli/sidecar.py | 31 +++++++++++++++++++++-- evals/proxy.py | 26 +++++++++++++------ tests/evals/test_proxy.py | 49 ++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 8935af00..6c369296 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -36,6 +36,31 @@ def __iter__(self) -> Iterator[Any]: yield self.call_source +def _pumps_blocking(meta: dict[str, Any] | None) -> bool: + """Whether a still-running pump means the recording may be short. + + Not every live pump is evidence of loss. A CLI that signals its MCP servers on exit + leaves the proxy's stdin read parked on a client that will never write again — the + trace is complete with respect to everything that client actually sent, and an + in-flight request that never got its answer is already counted as an unmatched + response. What does imply loss is a live *output* pump, because the server may have + been mid-reply when the deadline expired. + + Sidecars written before the proxy recorded per-stream detail carry only the boolean, + so they keep the old, stricter reading rather than being reinterpreted after the fact. + """ + if meta is None: + return False + if not meta.get("pumps_alive"): + return False + streams = meta.get("pumps_alive_streams") + if not isinstance(streams, list): + return True + if {"stdout", "stderr"} & {str(name) for name in streams}: + return True + return str(meta.get("finalization_reason") or "") not in ("signal", "child_exit") + + def _nonnegative_int(value: Any) -> int | None: if isinstance(value, bool) or not isinstance(value, int) or value < 0: return None @@ -231,6 +256,7 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] "unexpected_seq": 0, "invalid_meta_fields": 0, "pumps_alive": bool(meta.get("pumps_alive")) if meta is not None else False, + "pumps_blocking": _pumps_blocking(meta), "last_seq": last_seq, "tool_request_count": tool_request_count, } @@ -263,7 +289,7 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] or segment["torn_line"] or segment["skipped_rows"] > 0 or any((segment.get(key) or 0) > 0 for key in fatal_counts) - or segment["pumps_alive"] + or segment["pumps_blocking"] else "complete" ) session_calls.sort( @@ -287,6 +313,7 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] status["proxy_meta_not_final"] = any(segment["proxy_meta_not_final"] for segment in session_statuses) status["torn_line"] = any(segment["torn_line"] for segment in session_statuses) status["pumps_alive"] = any(segment["pumps_alive"] for segment in session_statuses) + status["pumps_blocking"] = any(segment["pumps_blocking"] for segment in session_statuses) aggregate_keys = ( "skipped_rows", *counter_keys, @@ -367,7 +394,7 @@ def _incompleteness_note(status: dict[str, Any]) -> str: value = status.get(key) if value and not (key == "proxy_meta_count" and value == 1): parts.append(f"{key}={int(value)}") - if status.get("pumps_alive"): + if status.get("pumps_blocking"): parts.append("pumps_alive=1") return ":".join(parts) diff --git a/evals/proxy.py b/evals/proxy.py index e1d618a7..8b4000d6 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -182,6 +182,10 @@ def __init__( self.server_requests = 0 self.child_killed = False self.pumps_alive = False + # Which streams were still pumping at finalization. A bare boolean cannot + # distinguish "the server may still have been talking to us" from "the client + # went away while our stdin read was parked", which are different facts. + self.pumps_alive_streams: set[str] = set() self.finalization_reason = "direct" self.finalization_signal: str | None = None self.finalized = False @@ -366,6 +370,7 @@ def write_meta(self) -> None: "tool_request_count": self._seq, "child_killed": self.child_killed, "pumps_alive": self.pumps_alive, + "pumps_alive_streams": sorted(self.pumps_alive_streams), "finalization_reason": self.finalization_reason, "finalization_signal": self.finalization_signal, "evidence_trace_available": self.evidence_active, @@ -767,10 +772,14 @@ def run_proxy( if rem > 0 and t_err is not None: t_err.join(timeout=rem) - pumps_still = any(t is not None and t.is_alive() for t in (t_in, t_out, t_err)) or not all( - e.is_set() if e is not None else True for e in (stdin_done, stdout_done, stderr_done) - ) - recorder.pumps_alive = pumps_still + for name, thread, done in ( + ("stdin", t_in, stdin_done), + ("stdout", t_out, stdout_done), + ("stderr", t_err, stderr_done), + ): + if (thread is not None and thread.is_alive()) or (done is not None and not done.is_set()): + recorder.pumps_alive_streams.add(name) + recorder.pumps_alive = bool(recorder.pumps_alive_streams) return map_child_returncode(child.returncode) except KeyboardInterrupt: # Preserve Python's existing SIGINT behaviour: unwind through the @@ -793,10 +802,11 @@ def run_proxy( pass # If pumps are still alive at deadline, note it; meta is still last row # (finalized flag drops any further appends from daemon pumps). - if t_out is not None or t_err is not None or t_in is not None: - still = any(t is not None and t.is_alive() for t in (t_in, t_out, t_err)) - if still: - recorder.pumps_alive = True + for name, thread in (("stdin", t_in), ("stdout", t_out), ("stderr", t_err)): + if thread is not None and thread.is_alive(): + recorder.pumps_alive_streams.add(name) + if recorder.pumps_alive_streams: + recorder.pumps_alive = True requested_signal = termination_signal() if termination_signal is not None else None if requested_signal is not None: _record_signal_finalization(recorder, requested_signal) diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 72899afd..7391adbe 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -11,6 +11,7 @@ import textwrap import time from pathlib import Path +from typing import Any import pytest @@ -22,6 +23,7 @@ ) from evals.core.results import AgentRun, TaskResult, agent_run_to_task_result from evals.drivers.cli.sidecar import ( + _pumps_blocking, apply_proxy_sidecar, ensure_proxy_pythonpath, load_proxy_sidecar, @@ -1961,3 +1963,50 @@ def test_a_response_the_agent_never_received_is_not_authoritative_evidence(tmp_p _calls, status = load_proxy_sidecar(tmp_path / "undelivered.jsonl") assert status["undelivered_lines"] == 1 assert status["state"] == "incomplete" + + +# --------------------------------------------------------------------------- +# Which pump is alive at shutdown decides whether a trace is short or merely +# interrupted. Claude Code signals its MCP servers on exit, so the proxy's stdin +# read is parked on a client that will never write again — every claude-cli row +# was charged to infrastructure for a pump that could not have lost anything. +# --------------------------------------------------------------------------- + + +def _meta(**kw: Any) -> dict[str, Any]: + base = {"pumps_alive": True, "pumps_alive_streams": ["stdin"], "finalization_reason": "signal"} + base.update(kw) + return base + + +@pytest.mark.parametrize( + ("meta", "blocking", "why"), + [ + pytest.param(None, False, "no meta row at all", id="no-meta"), + pytest.param( + _meta(pumps_alive=False, pumps_alive_streams=[]), False, "nothing was pumping", id="quiet-shutdown" + ), + # The case that made every claude-cli run an infra error. + pytest.param(_meta(), False, "client signalled away; stdin cannot deliver more", id="stdin-parked-on-signal"), + pytest.param( + _meta(finalization_reason="child_exit"), False, "server gone; same reasoning", id="stdin-on-child-exit" + ), + # A live stdin pump with no reason for the client to have stopped is still suspect. + pytest.param( + _meta(finalization_reason="normal_eof"), True, "EOF should have ended the read", id="stdin-after-clean-eof" + ), + # Output pumps carry the server's replies, so a live one may mean a lost response. + pytest.param(_meta(pumps_alive_streams=["stdout"]), True, "server may have been mid-reply", id="stdout-alive"), + pytest.param(_meta(pumps_alive_streams=["stderr"]), True, "same for stderr", id="stderr-alive"), + pytest.param( + _meta(pumps_alive_streams=["stdin", "stdout"]), True, "one bad stream is enough", id="mixed-streams" + ), + # Sidecars written before per-stream detail keep the old stricter reading rather + # than being silently reinterpreted in their favour. + pytest.param( + {"pumps_alive": True, "finalization_reason": "signal"}, True, "legacy file", id="pre-detail-sidecar" + ), + ], +) +def test_only_a_pump_that_could_have_lost_something_invalidates_the_trace(meta, blocking, why): + assert _pumps_blocking(meta) is blocking, why From c3d3a7157e21902538caff77d3b05840f5c97635 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Wed, 19 Aug 2026 10:13:12 +0530 Subject: [PATCH 68/93] Stop handing the agent a shell to route around the surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An eval of a tool surface gave the agent Claude Code's full built-in set alongside the 28 Plane tools. permission_mode is bypassPermissions, so the --allowedTools=mcp__plane__* argument granted auto-approval without restricting anything, and Bash ran unchallenged. Measured on a haiku subset: 11 of 24 repetitions made zero MCP calls. Each searched for a tool, did not call it, then spent its remaining turns in the shell. Ten exhausted the 15-turn budget with an empty answer. The eleventh succeeded — it added sys.path to the repo it was standing in, imported plane_mcp, ran `env | grep -i plane`, found the API key and called Plane's REST API directly. The work item really was added to the cycle, so the task verified as passed with no MCP call recorded at all. The driver now passes --tools= so the agent keeps none of Claude Code's own tools. That removes the bypass rather than merely detecting it after the fact, and it has a second effect: with the built-ins gone the total tool count falls under the threshold that defers MCP tools behind ToolSearch, so the surface arrives directly, the way every other driver already sees it. builtin_tools=None restores Claude Code's defaults for anyone who wants to measure the agent product rather than the surface. --- evals/drivers/cli/claude.py | 12 ++++++++++++ tests/evals/drivers/test_cli_driver.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py index 6632e8d7..8be82618 100644 --- a/evals/drivers/cli/claude.py +++ b/evals/drivers/cli/claude.py @@ -344,6 +344,7 @@ def __init__( python_bin: str | None = None, permission_mode: str = "bypassPermissions", strict_mcp: bool = True, + builtin_tools: str | None = "", runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, server_command: list[str] | None = None, use_proxy: bool = True, @@ -352,6 +353,14 @@ def __init__( self.claude_bin = claude_bin self.permission_mode = permission_mode self.strict_mcp = strict_mcp + # Which of Claude Code's own tools the agent keeps. Empty means none, which is + # what an eval of a *tool surface* wants: with Bash and Read in hand a model that + # cannot work the surface out reads the repo it is standing in, harvests the API + # key and calls Plane's REST API directly — measured, not hypothesised. Removing + # the built-ins also drops the total tool count under the threshold that defers + # MCP tools behind ToolSearch, so the surface arrives directly, as it does for + # every other driver. None keeps Claude Code's default set. + self.builtin_tools = builtin_tools # Full replacement for the MCP server launch (external surfaces under # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. super().__init__( @@ -412,6 +421,9 @@ def build_command( ] if self.strict_mcp: command.append("--strict-mcp-config") + if self.builtin_tools is not None: + # `=` form: --tools is variadic and would otherwise swallow the trailing prompt. + command.append(f"--tools={self.builtin_tools}") if model: command.extend(["--model", model]) if system: diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index bfa317c3..e3ccd090 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -1621,3 +1621,28 @@ def test_codex_nonzero_exit_is_not_scored_as_a_finished_attempt(tmp_path: Path): notes=[], ) assert clean.stopped_reason == "end_turn" + + +def _claude_command(**kw): + driver = ClaudeCliDriver(**kw) + launch = CliLaunch(cwd=Path("/tmp"), config_args=["--mcp-config", "/tmp/cfg.json"]) + return driver.build_command("do the task", model="haiku", max_turns=8, system=None, launch=launch) + + +def test_the_agent_gets_no_builtin_tools_by_default(): + """An eval of a tool surface must not hand the agent a shell to route around it. + + Measured: with Bash available, a model that could not work the surface out read the + repo it was standing in, harvested the API key from its own environment and called + Plane's REST API directly — the task verified as passed with zero MCP calls. + """ + command = _claude_command() + assert "--tools=" in command, command + # The `=` form matters: --tools is variadic and the bare form eats the prompt. + assert not any(part == "--tools" for part in command), command + assert command[-1] == "do the task", command[-1] + + +def test_builtin_tools_can_be_restored_or_named(): + assert "--tools=" not in _claude_command(builtin_tools=None) + assert "--tools=Bash,Read" in _claude_command(builtin_tools="Bash,Read") From 4ddc39cd7d6122c62139e85548d666952ba89608 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Wed, 19 Aug 2026 11:04:34 +0530 Subject: [PATCH 69/93] Score the surface, not the database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes that both let a claude-cli run be read honestly. A session that never called tools/list was counted as disagreeing about the manifest. Claude Code lists tools in one session and makes its calls in a second, so the quiet one discarded the fingerprint on nearly every row and the reporter refused the file: its rows could not be shown to have hit the same surface. Silence is not contradiction, and two sessions naming different fingerprints is already caught by len(unique) > 1. Write verifiers read Plane back, which answers "did the state change", not "did the agent change it through the surface under test". Measured: an agent that could not work the tools out added the repo it was standing in to sys.path, took the API key from its own environment and mutated Plane over REST. The work item really was added to the cycle, so the task scored a pass with no tool call recorded at all. A write task that changed Plane with no successful tool call now fails and says so. Only when the trace is trustworthy — where integrity is false the row is already an infrastructure error and zero calls means the recording failed rather than the agent skipping the surface. --- evals/drivers/cli/sidecar.py | 12 ++++--- evals/runner/live.py | 25 ++++++++++++++- tests/evals/runner/test_live.py | 56 +++++++++++++++++++++++++++++++++ tests/evals/test_proxy.py | 31 ++++++++++++++++++ tests/tools/test_governance.py | 1 + 5 files changed, 120 insertions(+), 5 deletions(-) diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index 6c369296..cb54cde7 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -330,10 +330,14 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] missing_manifests = len(metas) - len(manifests) status["tool_manifest_fingerprints"] = unique_manifests status["tool_manifest_missing_sessions"] = missing_manifests - status["tool_manifest_disagreement"] = len(unique_manifests) > 1 or bool(unique_manifests and missing_manifests) - status["tool_manifest_fingerprint"] = ( - unique_manifests[0] if len(unique_manifests) == 1 and missing_manifests == 0 else None - ) + # A session that never called tools/list has no opinion about the manifest, and silence + # is not contradiction. Claude Code splits the work — one session lists the tools, a + # second makes the calls and never lists — so counting the quiet one as a dissenter + # discarded the fingerprint on almost every row and left the reporter unable to + # establish that a file's rows hit the same surface. Real disagreement is two sessions + # reporting different fingerprints, which len(unique) > 1 already catches. + status["tool_manifest_disagreement"] = len(unique_manifests) > 1 + status["tool_manifest_fingerprint"] = unique_manifests[0] if len(unique_manifests) == 1 else None status["evidence_trace_available"] = ( bool(metas) and len(metas) == len(session_statuses) diff --git a/evals/runner/live.py b/evals/runner/live.py index 894f8258..3956a412 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -15,7 +15,7 @@ from evals.core.evidence import configured_evidence_labels from evals.core.results import TaskResult, agent_run_to_task_result from evals.core.server_env import stdio_server_env -from evals.core.task_metadata import build_task_metadata +from evals.core.task_metadata import build_task_metadata, entry_requires_mutation from evals.drivers import KNOWN_DRIVERS, get_driver from evals.drivers.api import MODEL_TIERS from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys @@ -362,6 +362,28 @@ async def _verify_task( ) -> None: """Run one verifier and record task outcomes or verifier failures.""" verify = task["verify"] + + def _require_surface_for_mutation(task: dict, agent: Any, *, ok: bool, note: str) -> tuple[bool, str]: + """A write task that changed Plane without calling a tool did not demonstrate the surface. + + Write verifiers read Plane back, so they answer "did the state change", not "did the + agent change it through the surface under test". Measured: an agent that could not + work the tools out read the repo it was standing in, took the API key from its own + environment and mutated Plane over REST. The state was correct and the task scored a + pass with no tool call recorded. + + Only applied to a trustworthy trace — when integrity is false the row is already an + infrastructure error, and zero calls there means the recording failed, not the agent. + """ + if not ok or not entry_requires_mutation({"tags": task.get("tags") or ()}): + return ok, note + if agent.trace_integrity is False: + return ok, note + calls = [call for call in (agent.to_row().get("calls") or []) if not bool(call.get("is_error"))] + if calls: + return ok, note + return False, f"{note}; surface=missing (write task changed Plane with 0 successful tool calls)" + try: agent_row = agent.to_row() ok, note = await verify( @@ -378,6 +400,7 @@ async def _verify_task( "trace_integrity_reason": agent.trace_integrity_reason, }, ) + ok, note = _require_surface_for_mutation(task, agent, ok=bool(ok), note=note) row.success = bool(ok) row.verify_note = note print( diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py index cd58a1c7..abdf30f3 100644 --- a/tests/evals/runner/test_live.py +++ b/tests/evals/runner/test_live.py @@ -1259,3 +1259,59 @@ def test_elapsed_formats_minutes_then_hours(monkeypatch): assert _elapsed(1000.0) == "01:15" clock["now"] = 1000.0 + 3671 assert _elapsed(1000.0) == "1:01:11" + + +@pytest.mark.parametrize( + ("tags", "calls", "expect_pass", "why"), + [ + pytest.param({"write"}, [], False, "mutation with no tool call", id="write-with-zero-calls"), + pytest.param({"write"}, [{"tool": "cycle"}], True, "mutation through the surface", id="write-with-calls"), + # An errored call is not use of the surface. + pytest.param( + {"write"}, [{"tool": "cycle", "is_error": True}], False, "only failed calls", id="write-all-errored" + ), + # Non-write tasks are gated by answer_with_provenance instead; do not double-charge them. + pytest.param({"schema"}, [], True, "not a write task", id="non-write-untouched"), + ], +) +def test_a_write_task_must_change_plane_through_the_surface(monkeypatch, tmp_path, tags, calls, expect_pass, why): + """An off-surface mutation verifies as correct state; it must not score as a pass. + + Measured: an agent read the repo it was standing in, took the API key from its own + environment and mutated Plane over REST. The verifier read the state back, found it + correct, and the row passed with zero tool calls recorded. + """ + from evals.runner import live as runner_live + + out = tmp_path / "out.jsonl" + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (MagicMock(), "test-ws")) + + def ok_seed(plane, run_id, needs, ctx, task_id=None): + ctx.update({"project_name": "EVAL surface", "project_id": "p1"}) + + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + + class SurfacelessDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=list(calls), + final_text="done", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + trace_integrity=True, + ) + + async def verify(*args, **kwargs): + return True, "state is correct" + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: SurfacelessDriver()) + task = {"id": "T9", "prompt": "mutate {project}", "tags": tags, "needs": set(), "verify": verify} + asyncio.run(run_live([task], model_alias="haiku", reps=1, label="l", out_path=out, driver_name="claude-cli")) + row = [json.loads(line) for line in out.read_text().splitlines() if json.loads(line).get("row_type") != "meta"][0] + assert row["success"] is expect_pass, f"{why}: {row.get('verify_note') or row.get('error')}" + if not expect_pass: + assert "surface=missing" in row["verify_note"], row["verify_note"] diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py index 7391adbe..f72c32f5 100644 --- a/tests/evals/test_proxy.py +++ b/tests/evals/test_proxy.py @@ -2010,3 +2010,34 @@ def _meta(**kw: Any) -> dict[str, Any]: ) def test_only_a_pump_that_could_have_lost_something_invalidates_the_trace(meta, blocking, why): assert _pumps_blocking(meta) is blocking, why + + +@pytest.mark.parametrize( + ("metas", "fingerprint", "disagreement", "why"), + [ + pytest.param([{"tool_manifest_fingerprint": "fp1"}], "fp1", False, "one session, one listing", id="single"), + # Claude Code lists tools in one session and makes the calls in another. The quiet + # session has no opinion, and counting it as a dissenter discarded the fingerprint + # on nearly every row and left the reporter unable to compare the file. + pytest.param( + [{"tool_manifest_fingerprint": "fp1"}, {}], "fp1", False, "abstaining session", id="one-lists-one-calls" + ), + pytest.param([{}, {}], None, False, "nobody listed", id="no-listing-anywhere"), + # Real disagreement is two sessions naming different surfaces. + pytest.param( + [{"tool_manifest_fingerprint": "fp1"}, {"tool_manifest_fingerprint": "fp2"}], + None, + True, + "genuinely different surfaces", + id="conflicting", + ), + ], +) +def test_a_session_that_never_listed_tools_does_not_veto_the_manifest(tmp_path, metas, fingerprint, disagreement, why): + base = tmp_path / "sidecar.jsonl" + for index, meta in enumerate(metas): + row = {"row_type": "proxy_meta", "finalized": True, "last_seq": 0, "tool_request_count": 0, **meta} + (tmp_path / f"sidecar.jsonl.{index}.jsonl").write_text(json.dumps(row) + "\n") + _calls, status = load_proxy_sidecar(base) + assert status["tool_manifest_fingerprint"] == fingerprint, why + assert status["tool_manifest_disagreement"] is disagreement, why diff --git a/tests/tools/test_governance.py b/tests/tools/test_governance.py index 3c59691c..1e5521e7 100644 --- a/tests/tools/test_governance.py +++ b/tests/tools/test_governance.py @@ -252,6 +252,7 @@ def test_the_feature_toggles_the_sdk_offers_are_all_reachable(): missing_flags = set(ProjectFeature.model_fields) - declared assert not missing_flags, f"ProjectFeature flags with no way to set them: {sorted(missing_flags)}" + PROPERTY_REFUSAL = HttpError( "Bad Request", status_code=400, response={"error": "This resource is managed at the workspace level"} ) From 5c88274e1b5d69472417a8437df8f9df5866e227 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Wed, 19 Aug 2026 15:38:23 +0530 Subject: [PATCH 70/93] Give seeded projects names with nothing id-shaped in them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The project name is everything an agent gets: it is told the name and must resolve it to a UUID, because project_id is required by 121 of the 183 actions. A weaker model skipped that step and submitted a substring of the name as the id instead. "EVAL 3c128f21" got sent verbatim as project_id. An earlier attempt at this moved the hex into parentheses — "EVAL Delivery Planning (3c128f21)" — and made it worse rather than better: the parentheses turned the hex into a cleaner token to extract, and non-UUID project_id attempts went from 4 to 17 across six repetitions. That commit was reverted. The hex was the bait in both shapes, so the name no longer carries any. A word drawn deterministically from the run prefix keeps names distinct enough that a leftover project from a crashed run cannot make a name lookup ambiguous, and keeps R6's two projects apart — its expected answer is a project name. Teardown deletes by recorded project_id, so uniqueness in the name was never needed for correctness, and `evals.cleanup --prefix "EVAL "` still matches. Real projects are not named after UUID fragments. A fixture that invites a confusion the surface would never meet in production charges the model for the harness's own choice. --- evals/core/fixtures.py | 102 ++++++++++++++++++++++++++++++++++ evals/seed/build.py | 3 +- evals/seed/plan.py | 7 ++- evals/seed/projects.py | 3 +- tests/evals/seed/test_seed.py | 33 +++++++++++ 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/evals/core/fixtures.py b/evals/core/fixtures.py index c78a6c67..9050ccf4 100644 --- a/evals/core/fixtures.py +++ b/evals/core/fixtures.py @@ -86,7 +86,109 @@ def is_evaluation_customer_name(name: str | None) -> bool: return (name or "").strip().casefold() in _EVALUATION_CUSTOMER_NAMES +# Project names. Nothing here may look like an id. +# +# Seeded projects used to be called "EVAL 3c128f21". An agent is told only the project name +# and has to resolve it to a UUID, since project_id is required by 121 of the 183 actions — +# and a weaker model skipped the resolution and submitted a hex-looking substring as the id. +# Moving the hex into parentheses made it worse, not better: it became a cleaner token to +# extract, and non-UUID project_id attempts went from 4 to 17 across six repetitions. +# +# So the name carries no hex at all. Teardown deletes by recorded project_id, so per-run +# uniqueness in the *name* is not required for correctness; the word suffix exists only so a +# leftover project from a crashed run cannot make an agent's name lookup ambiguous, and so R6 +# can tell its two projects apart. `python -m evals.cleanup --prefix "EVAL "` still matches. +EVAL_PROJECT_PREFIX = "EVAL " +PROJECT_TITLES = ("Delivery Planning", "Platform Migration") +# 64 words: one per run, derived from the seed so a name is reproducible from its run id. +PROJECT_SUFFIX_WORDS = ( + "Kestrel", + "Osprey", + "Falcon", + "Harrier", + "Merlin", + "Kite", + "Buzzard", + "Goshawk", + "Heron", + "Egret", + "Curlew", + "Plover", + "Godwit", + "Dunlin", + "Sanderling", + "Turnstone", + "Petrel", + "Fulmar", + "Gannet", + "Guillemot", + "Razorbill", + "Puffin", + "Skua", + "Tern", + "Swift", + "Martin", + "Swallow", + "Wagtail", + "Pipit", + "Dipper", + "Wren", + "Dunnock", + "Redstart", + "Whinchat", + "Wheatear", + "Fieldfare", + "Redwing", + "Blackcap", + "Chiffchaff", + "Firecrest", + "Treecreeper", + "Nuthatch", + "Jackdaw", + "Chough", + "Raven", + "Rook", + "Magpie", + "Jay", + "Linnet", + "Twite", + "Redpoll", + "Siskin", + "Crossbill", + "Hawfinch", + "Brambling", + "Yellowhammer", + "Corncrake", + "Lapwing", + "Woodcock", + "Snipe", + "Avocet", + "Oystercatcher", + "Shelduck", + "Wigeon", +) + + +def eval_project_name(run_prefix: str, *, second: bool = False) -> str: + """Build a seeded project's display name: readable, and never id-shaped. + + Deterministic in ``run_prefix`` so the same run always produces the same name, which + keeps a resumed run and its teardown in agreement. + """ + try: + index = int(str(run_prefix)[:8], 16) + except ValueError: + index = sum(ord(ch) for ch in str(run_prefix)) + word = PROJECT_SUFFIX_WORDS[index % len(PROJECT_SUFFIX_WORDS)] + title = PROJECT_TITLES[1 if second else 0] + return f"{EVAL_PROJECT_PREFIX}{title} {word}" + + __all__ = [ + "EVAL_PROJECT_PREFIX", + "PROJECT_SUFFIX_WORDS", + "PROJECT_TITLES", + "eval_project_name", "BLOCKING_REFERENCE_ADDRESS", "BLOCKING_SOURCE_TITLE", "BLOCKING_TARGET_TITLE", diff --git a/evals/seed/build.py b/evals/seed/build.py index a2e6ee95..09caf5b7 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -8,6 +8,7 @@ from plane import PlaneClient from evals.core.errors import TaskSkipped +from evals.core.fixtures import eval_project_name from .customers import ( CUSTOMER_NAME, @@ -270,7 +271,7 @@ def seed( even if a later fixture step raises (F5). """ run_prefix = run_id[:8] - project_name = f"EVAL {run_prefix}" + project_name = eval_project_name(run_prefix) workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] # Reset known keys while preserving object identity for the caller. diff --git a/evals/seed/plan.py b/evals/seed/plan.py index ce9b2834..f42edbfd 100644 --- a/evals/seed/plan.py +++ b/evals/seed/plan.py @@ -18,7 +18,7 @@ def seed_plan(needs: set[str]) -> list[str]: """Human-readable seed plan for --dry-run (no network).""" lines = [ - "project: EVAL {run8} (identifier EV{XXXX})", + "project: EVAL Delivery Planning {Word} (identifier EV{XXXX})", ] if "items" in needs: lines.append(f"items: {len(WORK_ITEM_FIXTURES)} work items (read truth randomised per row; default 4 urgent)") @@ -52,7 +52,10 @@ def seed_plan(needs: set[str]) -> list[str]: if "release" in needs: lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") if "second_project" in needs: - lines.append("second_project: EVAL {run8} B with random unequal open Bug counts across both projects (R6)") + lines.append( + "second_project: EVAL Platform Migration {Word} " + "with random unequal open Bug counts across both projects (R6)" + ) if "leave_cycles_worklogs_off" in needs: lines.append( "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 90b8d722..7160977c 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -12,6 +12,7 @@ from plane.models.workspaces import WorkspaceFeature from evals.core.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence +from evals.core.fixtures import eval_project_name from .gates import is_plan_gate from .identities import record_seeded_entity @@ -204,7 +205,7 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s from .item_types import seed_item_type run_prefix = context["run8"] - name = f"EVAL {run_prefix} B" + name = eval_project_name(run_prefix, second=True) project = create_project_with_identifier_retry( plane, workspace_slug, diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index beffb8ce..e4f46560 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -4,6 +4,7 @@ import asyncio import inspect +import re from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock @@ -1666,3 +1667,35 @@ def list(self, workspace_slug=None, params=None): ) def test_list_projects_behaviours(case): case() + + +def test_a_seeded_project_name_contains_nothing_that_looks_like_an_id(): + """The name is all an agent gets, and it must not offer a substring to submit as an id. + + Measured: with "EVAL 3c128f21" a weaker model sent project_id="EVAL 3c128f21" verbatim; + with "EVAL Delivery Planning (3c128f21)" it extracted the bare hex and sent that, 17 + times across six repetitions. The hex was the bait either way, so it is gone. + """ + from evals.core.fixtures import EVAL_PROJECT_PREFIX, PROJECT_SUFFIX_WORDS, eval_project_name + + for run_prefix in ("3c128f21", "cad7f69b", "deadbeef", "00000000", "ffffffff"): + for second in (False, True): + name = eval_project_name(run_prefix, second=second) + assert name.startswith(EVAL_PROJECT_PREFIX), f"cleanup --prefix must still match: {name}" + # No hex run of 6+ characters, which is what the model was pattern-matching on. + assert not re.search(r"\b[0-9a-f]{6,}\b", name, re.I), name + # No bare digits at all: a number is the other thing an id looks like. + assert not re.search(r"\d", name), name + assert name.split()[-1] in PROJECT_SUFFIX_WORDS, name + + # Deterministic in the run prefix, so a resumed run and its teardown agree on the name. + assert eval_project_name("3c128f21") == eval_project_name("3c128f21") + # The two projects stay distinguishable — R6's answer is a project name. + assert eval_project_name("3c128f21") != eval_project_name("3c128f21", second=True) + + +def test_a_non_hex_run_prefix_still_produces_a_name(): + """Never raise on the naming path: a fixture seed id shape change must not break seeding.""" + from evals.core.fixtures import eval_project_name + + assert eval_project_name("not-hex-at-all").startswith("EVAL ") From 56372c81f1265a3d36d1ec38660756b7aaae383a Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Wed, 19 Aug 2026 17:25:42 +0530 Subject: [PATCH 71/93] Make the Antigravity driver actually reach agy Two independent defects meant no agy row ever produced a measurement. The driver ran agy under an isolated HOME. agy keeps its OAuth token in the macOS login keychain, which Security resolves through $HOME/Library/Keychains, so the override made the keychain unfindable ("A keychain cannot be found to store \"antigravity\"") and every run failed unauthenticated. Isolate with the undocumented --gemini_dir instead: it relocates agy's whole state tree, which is all the isolation was ever for, and leaves the credential reachable. agy ignores a relative value and silently falls back to the real tree, so the path is resolved before use. The prompt was passed as a trailing positional after a bare -p. agy parses with Go's flag package, where --print (documented alias: --prompt) is a string flag that consumes the next argv entry -- so -p took "--output-format" as the prompt, the real prompt became a stray positional that ended flag parsing, and --dangerously-skip-permissions never took effect. agy answered a question about its own CLI and then denied its own tool calls. Every string flag now uses the --flag=value form, with --print last, so argument order cannot matter. Also drops the invoke_cli override: it existed to retry without env= for injected test runners, and nothing reaches it now. Verified live against eval-surface-a: C1 alone, then C1,S1,S2,S5,W5,W6,I3,R6 on gemini-3.6-flash-low -- 8/8 rows evaluated, 7 pass, and the one failure is a wrong answer rather than an infra error. --- evals/DESIGN.md | 6 +- evals/README.md | 2 +- evals/drivers/cli/antigravity.py | 134 ++++++++++--------------- tests/evals/drivers/test_cli_driver.py | 30 +++--- tests/evals/drivers/test_vendors.py | 115 +++++++++++---------- 5 files changed, 132 insertions(+), 155 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 68d2c371..df3fa679 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -246,11 +246,11 @@ effective-config evidence differs by vendor: | Claude | **Readback-supported, not behaviorally proven for the evaluated invocation.** Real `claude mcp list` reads the same isolated `.claude.json` and observes only `plane`. The evaluated `claude -p` receives that file plus `--strict-mcp-config`; exclusion of project/ambient MCP servers rests on the CLI's documented strict-config contract, not a forbidden-server probe of that invocation. HOME, `CLAUDE_CONFIG_DIR`, and all XDG roots are isolated. | | Codex | Proven by real `codex mcp list --json` readback under the isolated Codex home. | | OpenCode | Proven by real `opencode debug config` readback under isolated HOME/XDG roots and the generated project config. | -| Antigravity | **Unverifiable.** Antigravity CLI 1.1.13 has no MCP/effective-config introspection command. The harness isolates HOME/XDG roots and inspects generated files, but neither the harness nor this design treats that as observed effective-config exclusivity. | +| Antigravity | **Unverifiable.** Antigravity CLI has no MCP/effective-config introspection command. The harness relocates agy's whole state tree with the undocumented `--gemini_dir` and inspects generated files, but neither the harness nor this design treats that as observed effective-config exclusivity. HOME is deliberately *not* isolated: agy keeps its OAuth token in the macOS login keychain, which Security resolves under `$HOME/Library/Keychains`, so an isolated HOME made the credential unfindable and every run failed unauthenticated. | The Antigravity "unverifiable" regression test is documentation coverage: it guards this -claim, not runtime behavior. Separate behavioral tests cover HOME/XDG isolation and generated -file placement, but those still cannot observe Antigravity's effective server set. +claim, not runtime behavior. Separate behavioral tests cover the isolated gemini dir and +generated file placement, but those still cannot observe Antigravity's effective server set. Several loop rules are deliberately centralized in `ApiDriver`: diff --git a/evals/README.md b/evals/README.md index a5cf1ff9..6cfce5a0 100644 --- a/evals/README.md +++ b/evals/README.md @@ -66,7 +66,7 @@ errors, and observed tool distributions use the same rules as local-server rows. | `api` | Owned API + MCP loop | Provider-neutral; tiers resolve for `--provider anthropic` (default) or `openai` | | `codex-cli` | OpenAI Codex CLI | `standard` and `fast` resolve to verified GPT-5.6 IDs | | `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku`; isolated HOME/config/XDG roots and strict MCP config | -| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; isolated HOME/XDG and generated config, but 1.1.13 has no effective-config readback, so exclusivity is unverifiable | +| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; isolated via `--gemini_dir` with HOME left real (agy's token lives in the macOS login keychain), but agy has no effective-config readback, so exclusivity is unverifiable | | `opencode-cli` | OpenCode | Tiers are intentionally unmapped; pass an explicit ID listed by `opencode models` | ### Model tiers diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py index 1b83a910..e70e204b 100644 --- a/evals/drivers/cli/antigravity.py +++ b/evals/drivers/cli/antigravity.py @@ -36,51 +36,31 @@ def write_antigravity_mcp_config( path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") -def prepare_antigravity_fake_home( - fake_home: Path, +def prepare_antigravity_gemini_dir( + gemini_dir: Path, *, command: str, args: list[str], env: dict[str, str], - real_home: Path | None = None, ) -> None: - """Build an isolated HOME for agy with MCP config plus copied auth artifacts. + """Build an isolated ``--gemini_dir`` tree for agy holding only our MCP server. - Writes mcp_config.json to both documented paths (~/.gemini/config/ and - ~/.gemini/antigravity-cli/) since which one agy reads is unsettled. antigravity-cli is - a real directory and the oauth token a plain copy, never symlinks — otherwise a token - refresh or a runtime log would write through into the user's real home. + Writes mcp_config.json to both paths agy reads under its gemini dir (``config/`` + and ``antigravity-cli/``) since which one wins is unsettled; agy creates an empty + ``config/mcp_config.json`` itself when none is present. + + This replaces an isolated HOME. agy keeps its OAuth token in the macOS login + keychain, which Security resolves through ``$HOME/Library/Keychains`` — so + overriding HOME made the keychain unfindable ("A keychain cannot be found to store + \"antigravity\"") and every run failed unauthenticated. ``--gemini_dir`` moves only + agy's own state, leaving HOME real and the credential reachable. """ - real_home = real_home or Path.home() - gemini_root = fake_home / ".gemini" - gemini_root.mkdir(parents=True, exist_ok=True) - - real_cli = real_home / ".gemini" / "antigravity-cli" - fake_cli = gemini_root / "antigravity-cli" - # Always a real directory — never symlink the whole tree. - if fake_cli.is_symlink() or fake_cli.is_file(): - fake_cli.unlink() - fake_cli.mkdir(parents=True, exist_ok=True) - - # Share auth via plain COPY only — never symlink (in-place token refresh - # must not write through into the real home). - if real_cli.is_dir(): - for name in ("antigravity-oauth-token",): - src = real_cli / name - dst = fake_cli / name - if src.is_file() and not dst.exists(): - try: - dst.write_bytes(src.read_bytes()) - except OSError: - pass - - # Dual write as real files (not through any symlink). for rel in ( - Path(".gemini") / "config" / "mcp_config.json", - Path(".gemini") / "antigravity-cli" / "mcp_config.json", + Path("config") / "mcp_config.json", + Path("antigravity-cli") / "mcp_config.json", ): write_antigravity_mcp_config( - fake_home / rel, + gemini_dir / rel, command=command, args=args, env=env, @@ -92,12 +72,18 @@ class AntigravityCliDriver(CliDriver): """Run tasks via Google Antigravity CLI (``agy``). Probed 2026-08-12: -p headless, --output-format text|json|stream-json, --model, - --dangerously-skip-permissions. MCP only via ~/.gemini/config/mcp_config.json with no - CLI flag, hence HOME isolation; no turn-cap flag, so hit_max_turns=False plus a note. - Tool calls come from the proxy sidecar, not from parsing agy stdout. Antigravity CLI - 1.1.13 has no MCP or effective-config introspection command, so its server exclusivity - cannot be proven by real-binary readback: it is explicitly unverifiable and supported - only by isolated HOME/XDG roots plus inspection of the generated files. + --dangerously-skip-permissions. No turn-cap flag, so hit_max_turns=False plus a note. + Tool calls come from the proxy sidecar, not from parsing agy stdout. + + MCP config is not a flag, so the config must be planted somewhere agy will read. + Re-probed 2026-08-19 on 1.1.15: the undocumented ``--gemini_dir`` relocates agy's + whole state tree, which isolates the config without touching HOME. It must be an + absolute path — agy logs "must be an absolute path" and silently falls back to the + real one otherwise, which would hand the agent the user's own servers. + + Antigravity CLI has no MCP or effective-config introspection command, so server + exclusivity still cannot be proven by real-binary readback: it rests on the isolated + gemini dir plus inspection of the generated files. """ name = "antigravity-cli" @@ -133,27 +119,24 @@ def write_mcp_config( server_command: list[str], child_env: dict[str, str], ) -> CliLaunch: - fake_home = temp_dir / "home" - prepare_antigravity_fake_home( - fake_home, + # Absolute, because agy ignores a relative --gemini_dir and falls back to the + # real one; temp_dir is already absolute but resolve() makes that a guarantee + # rather than a caller's promise. + gemini_dir = (temp_dir / "gemini").resolve() + prepare_antigravity_gemini_dir( + gemini_dir, command=server_command[0], args=server_command[1:], env=child_env, ) - xdg_roots = { - name: temp_dir / name.lower().replace("_home", "") - for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") - } - for directory in xdg_roots.values(): - directory.mkdir(parents=True, exist_ok=True) - run_env = { - **os.environ, - "HOME": str(fake_home), - **{name: str(directory) for name, directory in xdg_roots.items()}, - } + run_env = None if "PATH" in child_env: - run_env["PATH"] = child_env["PATH"] - return CliLaunch(cwd=task_cwd, env=run_env) + run_env = {**os.environ, "PATH": child_env["PATH"]} + return CliLaunch( + cwd=task_cwd, + config_args=[f"--gemini_dir={gemini_dir}"], + env=run_env, + ) def build_command( self, @@ -164,35 +147,28 @@ def build_command( system: str | None, launch: CliLaunch, ) -> list[str]: - del max_turns, launch + del max_turns full_prompt = prompt if not system else f"{system}\n\n{prompt}" + # Every string flag takes the ``--flag=value`` form. agy parses with Go's flag + # package, where a string flag consumes the next argv entry: written as + # ``-p `` with other flags after it, ``-p`` ate ``--output-format``, the + # real prompt became a stray positional that ended flag parsing, and + # --dangerously-skip-permissions never took effect. agy then answered a question + # about its own CLI and denied its own tool calls. Keeping value and flag in one + # argv entry makes the ordering irrelevant. command = [ self.agy_bin, - "-p", - "--output-format", - "json", + # --gemini_dir chooses the state tree agy reads everything else out of. + *launch.config_args, + "--output-format=json", "--dangerously-skip-permissions", ] if model: - command.extend(["--model", model]) - command.append(full_prompt) + command.append(f"--model={model}") + # Last, so nothing can be mistaken for its value. + command.append(f"--print={full_prompt}") return command - def invoke_cli( - self, - command: list[str], - *, - launch: CliLaunch, - timeout_s: int, - ) -> subprocess.CompletedProcess[str]: - try: - return super().invoke_cli(command, launch=launch, timeout_s=timeout_s) - except TypeError: - # Some test runners reject ``env=``; retry without it. A timeout - # from this fallback still reaches the template's harvest path. - fallback = CliLaunch(cwd=launch.cwd, config_args=launch.config_args) - return super().invoke_cli(command, launch=fallback, timeout_s=timeout_s) - def parse_output( self, proc: subprocess.CompletedProcess[str], @@ -220,6 +196,6 @@ def parse_output( __all__ = [ "AntigravityCliDriver", - "prepare_antigravity_fake_home", + "prepare_antigravity_gemini_dir", "write_antigravity_mcp_config", ] diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py index e3ccd090..54056403 100644 --- a/tests/evals/drivers/test_cli_driver.py +++ b/tests/evals/drivers/test_cli_driver.py @@ -700,14 +700,14 @@ def fake_run(cmd, **kwargs): if cfg.is_file(): bag["cfg"] = json.loads(cfg.read_text()) elif driver_cls is AntigravityCliDriver: - env = kwargs.get("env") or {} - home = env.get("HOME") - if home: + flag = next((a for a in cmd if a.startswith("--gemini_dir=")), None) + if flag: + gemini_dir = Path(flag.split("=", 1)[1]) for rel in ( - Path(".gemini") / "config" / "mcp_config.json", - Path(".gemini") / "antigravity-cli" / "mcp_config.json", + Path("config") / "mcp_config.json", + Path("antigravity-cli") / "mcp_config.json", ): - p = Path(home) / rel + p = gemini_dir / rel if p.is_file(): bag.setdefault("cfgs", []).append(json.loads(p.read_text())) elif driver_cls is CodexCliDriver: @@ -1013,7 +1013,7 @@ def test_opencode_isolated_environment_effective_mcp_server_list_is_exactly_plan [ pytest.param(ClaudeCliDriver, "claude_bin", id="claude-config"), pytest.param(CodexCliDriver, "codex_bin", id="codex-argv"), - pytest.param(AntigravityCliDriver, "agy_bin", id="antigravity-home-config"), + pytest.param(AntigravityCliDriver, "agy_bin", id="antigravity-gemini-dir-config"), pytest.param(OpencodeCliDriver, "opencode_bin", id="opencode-cwd-config"), ], ) @@ -1042,12 +1042,13 @@ def fake_run(cmd, **kwargs): server = config["mcp_servers"]["plane"] proxy_args = [server["command"], *server["args"]] elif Driver is AntigravityCliDriver: - fake_home = Path(kwargs["env"]["HOME"]) + flag = next(a for a in cmd if a.startswith("--gemini_dir=")) + gemini_dir = Path(flag.split("=", 1)[1]) for rel in ( - Path(".gemini/config/mcp_config.json"), - Path(".gemini/antigravity-cli/mcp_config.json"), + Path("config/mcp_config.json"), + Path("antigravity-cli/mcp_config.json"), ): - configs.append(json.loads((fake_home / rel).read_text())) + configs.append(json.loads((gemini_dir / rel).read_text())) proxy_args = configs[0]["mcpServers"]["plane"]["args"] else: config_path = Path(kwargs["cwd"]) / "opencode.json" @@ -1064,7 +1065,8 @@ def fake_run(cmd, **kwargs): assert set(configs[0]["mcp_servers"]) == {"plane"} elif Driver is AntigravityCliDriver: assert set(configs[0]["mcpServers"]) == {"plane"} - assert all(kwargs["env"].get(name) for name in ("HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME")) + # HOME is intentionally the real one here — see the keychain note on the driver. + assert Path(gemini_dir).is_absolute() else: assert set(configs[0]["mcp"]) == {"plane"} assert all(kwargs["env"].get(name) for name in ("HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME")) @@ -1129,8 +1131,8 @@ def test_antigravity_effective_config_exclusivity_is_documented_as_unverifiable( driver_doc = AntigravityCliDriver.__doc__ or "" design = (REPO / "evals" / "DESIGN.md").read_text(encoding="utf-8") - assert "1.1.13 has no MCP or effective-config introspection command" in driver_doc - assert "explicitly unverifiable" in driver_doc + assert "has no MCP or effective-config introspection command" in driver_doc + assert "cannot be proven by real-binary readback" in driver_doc assert "| Antigravity | **Unverifiable.**" in design assert "neither the harness nor this design treats that as observed effective-config exclusivity" in design assert 'The Antigravity "unverifiable" regression test is documentation coverage' in design diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py index a8013704..ac3e6258 100644 --- a/tests/evals/drivers/test_vendors.py +++ b/tests/evals/drivers/test_vendors.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -17,7 +18,7 @@ from evals.drivers.api.driver import ApiDriver from evals.drivers.cli.antigravity import ( AntigravityCliDriver, - prepare_antigravity_fake_home, + prepare_antigravity_gemini_dir, write_antigravity_mcp_config, ) from evals.drivers.cli.claude import ( @@ -923,16 +924,16 @@ def test_get_driver_api(): assert isinstance(get_driver("codex-cli"), CodexCliDriver) -def _antigravity_driver_writes_mcp_config_under_isolated_home(tmp_path): +def _antigravity_driver_isolates_via_gemini_dir_not_home(tmp_path): seen: dict = {} def fake_run(cmd, **kwargs): seen["cmd"] = cmd - env = kwargs.get("env") or {} - seen["env"] = env - home = env.get("HOME") - if home: - cfg = Path(home) / ".gemini" / "config" / "mcp_config.json" + seen["env"] = kwargs.get("env") or {} + flag = next((a for a in cmd if a.startswith("--gemini_dir=")), None) + seen["gemini_dir"] = flag + if flag: + cfg = Path(flag.split("=", 1)[1]) / "config" / "mcp_config.json" seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") @@ -945,17 +946,29 @@ def fake_run(cmd, **kwargs): cwd=tmp_path, ) assert seen["cmd"][0] == "agy" - assert "-p" in seen["cmd"] - assert "--output-format" in seen["cmd"] - assert "json" in seen["cmd"] - assert "--model" in seen["cmd"] and "gemini-2.5" in seen["cmd"] + assert "--output-format=json" in seen["cmd"] + assert "--dangerously-skip-permissions" in seen["cmd"] + assert "--model=gemini-2.5" in seen["cmd"] + # The prompt must be the VALUE of --print: agy parses with Go's flag package, so a + # bare "-p" followed by other flags eats the next flag as its prompt and drops the + # real one, taking --dangerously-skip-permissions down with it. + assert seen["cmd"][-1] == "--print=do it" + assert not any(a in ("-p", "--print") for a in seen["cmd"]) assert "no_turn_cap" in run.notes + # The flag carries an absolute path and precedes the -p subcommand flags. + assert seen["gemini_dir"] is not None + assert Path(seen["gemini_dir"].split("=", 1)[1]).is_absolute() + assert seen["cmd"].index(seen["gemini_dir"]) == 1 + # HOME must stay the real one: agy reads its OAuth token from the macOS login + # keychain, which Security resolves under $HOME/Library/Keychains. Overriding it + # made the keychain unfindable and every run failed unauthenticated. + assert seen["env"].get("HOME") == os.environ.get("HOME") assert seen.get("mcp_cfg") is not None assert "mcpServers" in seen["mcp_cfg"] assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) -def _antigravity_fallback_runner_timeout_harvests(tmp_path): +def _antigravity_timeout_still_harvests_proxy_rows(tmp_path): call_row = { "tool": "g_tool", "args": {}, @@ -975,26 +988,20 @@ def _antigravity_fallback_runner_timeout_harvests(tmp_path): } def fake_run(cmd, **kwargs): - run_env = kwargs.get("env") or {} - home = run_env.get("HOME") - if home: - # First attempt includes env= — plant sidecar from dual-written mcp config, - # then reject env so the driver retries without it. - for rel in ( - Path(home) / ".gemini" / "config" / "mcp_config.json", - Path(home) / ".gemini" / "antigravity-cli" / "mcp_config.json", - ): - if rel.is_file(): - cfg = json.loads(rel.read_text()) - args = cfg["mcpServers"]["plane"]["args"] - side = Path(args[args.index("--log") + 1]) - side.write_text( - "\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", - encoding="utf-8", - ) - break - raise TypeError("runner does not accept env=") - # Fallback call (no env) times out — outer except must still harvest. + # Plant the sidecar the way a real run would have, from whichever of the two + # dual-written configs is present, then time out. The rows must still be + # harvested: a timed-out agy has usually already made its calls. + flag = next(a for a in cmd if a.startswith("--gemini_dir=")) + gemini_dir = Path(flag.split("=", 1)[1]) + for rel in ( + gemini_dir / "config" / "mcp_config.json", + gemini_dir / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + args = json.loads(rel.read_text())["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text("\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", encoding="utf-8") + break raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) @@ -1014,8 +1021,8 @@ def fake_run(cmd, **kwargs): @pytest.mark.parametrize( "case", case_params( - _antigravity_driver_writes_mcp_config_under_isolated_home, - _antigravity_fallback_runner_timeout_harvests, + _antigravity_driver_isolates_via_gemini_dir_not_home, + _antigravity_timeout_still_harvests_proxy_rows, ), ) def test_antigravity_behaviours(case, tmp_path): @@ -1076,39 +1083,31 @@ def fake_run(cmd, **kwargs): assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) -def test_prepare_antigravity_fake_home_dual_write_and_auth_only(tmp_path: Path): +def test_prepare_antigravity_gemini_dir_dual_writes_and_leaves_home_alone(tmp_path: Path): real_home = tmp_path / "real" cli = real_home / ".gemini" / "antigravity-cli" cli.mkdir(parents=True) - token_path = cli / "antigravity-oauth-token" - token_path.write_text("secret", encoding="utf-8") - # Snapshot real home before setup — must be byte-identical after. + (cli / "antigravity-oauth-token").write_text("secret", encoding="utf-8") + (cli / "mcp_config.json").write_text('{"mcpServers": {"other": {}}}', encoding="utf-8") before = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} - fake = tmp_path / "fake" - prepare_antigravity_fake_home( - fake, + gemini_dir = tmp_path / "isolated" / "gemini" + prepare_antigravity_gemini_dir( + gemini_dir, command="python", args=["-m", "evals.proxy", "--log", "s", "--", "x"], env={"PLANE_API_KEY": "k"}, - real_home=real_home, ) - p1 = fake / ".gemini" / "config" / "mcp_config.json" - p2 = fake / ".gemini" / "antigravity-cli" / "mcp_config.json" + p1 = gemini_dir / "config" / "mcp_config.json" + p2 = gemini_dir / "antigravity-cli" / "mcp_config.json" assert p1.is_file() and p2.is_file() - fake_cli = fake / ".gemini" / "antigravity-cli" - assert fake_cli.is_dir() and not fake_cli.is_symlink() - # Auth artifact is a plain COPY — never a symlink (no write-through path). - token = fake_cli / "antigravity-oauth-token" - assert token.is_file() and not token.is_symlink() - assert token.read_text(encoding="utf-8") == "secret" - # Writing the fake token must not mutate the real one. - token.write_text("mutated", encoding="utf-8") - assert token_path.read_text(encoding="utf-8") == "secret" - # mcp_config is a real file in the fake tree, not inside real home. - assert not (cli / "mcp_config.json").exists() - data = json.loads(p1.read_text()) - assert data["mcpServers"]["plane"]["command"] == "python" - # Real home byte-for-byte untouched (including oauth token). + for path in (p1, p2): + data = json.loads(path.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + # Only our server — the user's own entries are not carried over. + assert list(data["mcpServers"]) == ["plane"] + # Nothing is copied out of the real home and nothing written into it: auth comes + # from the login keychain, which stays reachable because HOME is never moved. + assert not (gemini_dir / "antigravity-cli" / "antigravity-oauth-token").exists() after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} assert after == before From ff9e4b3dce41ad40881a21dc466a65a7e2db994a Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 20 Aug 2026 15:40:57 +0530 Subject: [PATCH 72/93] Split the errored-call count by what kind of "no" a call received One number answered three unrelated questions. On a 35-task battery against a low-tier model, 31 of 40 errored calls were the schema correcting a malformed call, one was a genuine tool-design defect, and one was a fair question fairly answered -- and reading them as a single 12.8% rate is what kept the defect invisible for three model families in a row. Errored calls are now classified where the payload still exists, in the proxy, storing the category and never the text: refused turned away without acting -- missing field, stray argument, value outside an enum. A property of the tool schema; the API was never asked. rejected well formed, and the API refused its meaning. Undocumented preconditions live here. This is the number to act on. not_found a read came back absent. denied credentials or plan. failed the server or transport broke. The report splits those into surface friction, navigation cost, and answered existence questions. A first not_found on a given tool and action is an answer rather than an obstacle -- `project_estimate retrieve` before creating one is the correct move, and three models were each charged friction for making it -- so only a repeat counts. `unclassified` is reported apart from `denied`/`failed` so that a split reading zero surface friction because nothing was classified cannot be mistaken for a surface with no friction; a run recorded before this field existed lands there in full. Classification reads HTTP status and the FastMCP validation shape, not this server's ACTIONS table, so a foreign surface still classifies: the same battery scored a 177-tool build today, and coupling to the catalogue would have ended that. Two patterns for this server's own refusal wording are additive. Verified live: a three-task run put one error in each column with none unclassified -- workspace refused, project_estimate.retrieve not_found, cycle.transfer_workitems rejected. The first attempt reported all of them unclassified, because the sidecar reader rebuilds each call from an explicit key list and dropped the field; that whole path is now asserted hop by hop, and the test fails against the reader that dropped it. --- evals/core/error_class.py | 117 ++++++++++++++++ evals/core/results.py | 6 + evals/drivers/api/driver.py | 5 + evals/drivers/cli/sidecar.py | 5 + evals/proxy.py | 5 + evals/report/schema_friction.py | 105 +++++++++++++- tests/evals/report/test_summary.py | 7 + tests/evals/report/test_table.py | 7 + tests/evals/test_error_class.py | 218 +++++++++++++++++++++++++++++ 9 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 evals/core/error_class.py create mode 100644 tests/evals/test_error_class.py diff --git a/evals/core/error_class.py b/evals/core/error_class.py new file mode 100644 index 00000000..e556adda --- /dev/null +++ b/evals/core/error_class.py @@ -0,0 +1,117 @@ +"""What kind of "no" a tool call received. + +One errored-call count answers three unrelated questions at once, and the answers +pull in different directions: + + refused the server turned the call away without acting on it -- a required + field missing, an argument the action does not take, a value outside + an enum. Entirely a property of the tool schema. The API was never + asked anything. + rejected the call was well formed and the API refused its meaning: an + undocumented precondition, a conflict. This is where tool-design + defects live, and it is the number worth reading. + not_found a read that came back absent. Usually an answer rather than an + obstacle -- "is there an estimate on this project?" has no cheaper + form than asking -- so it is reported apart from friction. + denied credentials or plan. Says nothing about the tool surface. + failed the server or transport broke. + +Classification runs where the payload still exists (the proxy), and stores only +the category, never the text. + +Deliberately not coupled to this server: the categories are read off HTTP status +and the FastMCP/Pydantic validation shape, both of which any MCP server over a +REST API produces. Two patterns for this server's own refusal wording are +additive -- a foreign surface that does not match them still classifies by +status. That is what let one battery score both a 28-tool and a 177-tool server. +""" + +from __future__ import annotations + +import re + +REFUSED = "refused" +REJECTED = "rejected" +NOT_FOUND = "not_found" +DENIED = "denied" +FAILED = "failed" +UNCLASSIFIED = "unclassified" + +ERROR_CLASSES = (REFUSED, REJECTED, NOT_FOUND, DENIED, FAILED, UNCLASSIFIED) + +#: Classes counted as friction attributable to the tool surface's design. +SURFACE_FRICTION_CLASSES = (REJECTED,) +#: Classes counted as the cost of navigating the schema rather than the API. +NAVIGATION_CLASSES = (REFUSED,) + +_STATUS = re.compile(r"\b(?:HTTP|status(?:[ _]code)?[:= ]*)\s*(\d{3})\b", re.IGNORECASE) + +# Shapes any FastMCP server emits when a call fails its own signature, before +# the tool body runs. +_VALIDATION = ( + "validation error for", + "missing required argument", + "input should be", + "unexpected keyword argument", +) + +# This server's own refusals, which are deliberate answers rather than failures +# of validation, so they carry no status and no pydantic shape. +_OWN_REFUSALS = ( + "requires an action. it takes:", + "does not take:", +) + +_BY_STATUS = { + 400: REJECTED, + 409: REJECTED, + 422: REJECTED, + 401: DENIED, + 402: DENIED, + 403: DENIED, + 404: NOT_FOUND, +} + + +def classify_error(payload: str | None) -> str: + """Return the category of a failed call from its error payload. + + Status wins over wording: a 404 whose body happens to mention a missing + argument is still an absent resource. Only when no status is present does the + validation shape decide, because that is the case where the call never + reached the API at all. + """ + text = (payload or "").strip() + if not text: + return UNCLASSIFIED + lowered = text.lower() + + match = _STATUS.search(text) + if match: + status = int(match.group(1)) + if status in _BY_STATUS: + return _BY_STATUS[status] + if 500 <= status <= 599: + return FAILED + if 400 <= status <= 499: + return REJECTED + + if any(marker in lowered for marker in _OWN_REFUSALS): + return REFUSED + if any(marker in lowered for marker in _VALIDATION): + return REFUSED + return UNCLASSIFIED + + +__all__ = [ + "ERROR_CLASSES", + "NAVIGATION_CLASSES", + "SURFACE_FRICTION_CLASSES", + "classify_error", + "DENIED", + "FAILED", + "NOT_FOUND", + "REFUSED", + "REJECTED", + "UNCLASSIFIED", +] diff --git a/evals/core/results.py b/evals/core/results.py index 163610fb..6e830e15 100644 --- a/evals/core/results.py +++ b/evals/core/results.py @@ -107,6 +107,8 @@ class CallRecord: duration_ms: float | int | None = None action: str | None = None raw_tool: str | None = None + # Which kind of "no" an errored call received; None when the call succeeded. + error_class: str | None = None result_tokens_skipped: str | None = None # None means the response was not checked; [] means checked with no match. observed_sentinels: list[str] | None = None @@ -272,6 +274,8 @@ def usage_row(item: Usage) -> dict[str, int]: item["action"] = call.action if call.result_tokens_skipped is not None: item["result_tokens_skipped"] = call.result_tokens_skipped + if call.error_class is not None: + item["error_class"] = call.error_class if call.observed_sentinels is not None: item["observed_sentinels"] = list(call.observed_sentinels) calls.append(item) @@ -364,6 +368,7 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: result_chars=int(raw.get("result_chars") or 0), result_kind=str(raw.get("result_kind") or "text"), is_error=bool(raw.get("is_error")), + error_class=(str(raw["error_class"]) if raw.get("error_class") is not None else None), result_tokens_estimated=( bool(raw["result_tokens_estimated"]) if raw.get("result_tokens_estimated") is not None else None ), @@ -569,6 +574,7 @@ def agent_run_to_task_result( result_chars=result_chars, result_kind=str(c.get("result_kind") or "text"), is_error=bool(c.get("is_error")), + error_class=(str(c["error_class"]) if c.get("error_class") is not None else None), result_tokens_estimated=bool(estimated), result_token_count_method=str(count_method), duration_ms=c.get("duration_ms"), diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 1c3b64fe..772e86e4 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -23,6 +23,7 @@ from mcp import ClientSession from mcp.client.stdio import StdioServerParameters, stdio_client +from evals.core.error_class import classify_error from evals.core.evidence import ( configured_evidence_labels, normalize_evidence_aggregates, @@ -352,6 +353,10 @@ async def _run_task( calls[idx]["result_kind"] = result.kind calls[idx]["is_error"] = result.is_error calls[idx]["duration_ms"] = duration_ms + if result.is_error: + # Same classification the proxy applies to CLI runs, so the + # two driver families produce comparable rows. + calls[idx]["error_class"] = classify_error(result.text) if evidence_active: aggregate_observations = observed_aggregates( result.text, diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py index cb54cde7..3807abb2 100644 --- a/evals/drivers/cli/sidecar.py +++ b/evals/drivers/cli/sidecar.py @@ -194,6 +194,11 @@ def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any] "duration_ms": row.get("duration_ms"), "seq": row.get("seq"), } + if isinstance(row.get("error_class"), str): + # The proxy classifies while the payload still exists; this reader is the + # only path from the sidecar to a result row, so a key absent here is a + # key that never reaches the report. + call["error_class"] = row["error_class"] if isinstance(row.get("result_text"), str): call["result_text"] = row["result_text"] if isinstance(row.get("observed_sentinels"), list): diff --git a/evals/proxy.py b/evals/proxy.py index 8b4000d6..e0d85bc8 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Any +from evals.core.error_class import classify_error from evals.core.evidence import ( EVIDENCE_SENTINELS_ENV, consume_evidence_config, @@ -327,6 +328,10 @@ def on_server_message(self, obj: dict[str, Any]) -> None: "duration_ms": duration_ms, "seq": pending["seq"], } + if is_error: + # Classified here because this is the last place the payload exists: + # rows keep result_chars, not the text. Only the category is stored. + row["error_class"] = classify_error(result_text) if self.evidence_active: # Persist only labels matched from non-enumerable sentinels and # target-bound aggregate values the agent already received. The diff --git a/evals/report/schema_friction.py b/evals/report/schema_friction.py index b6a2302f..853ccc3e 100644 --- a/evals/report/schema_friction.py +++ b/evals/report/schema_friction.py @@ -5,7 +5,8 @@ from collections import defaultdict from dataclasses import dataclass -from evals.core.results import TaskResult +from evals.core.error_class import NOT_FOUND, REFUSED, REJECTED, UNCLASSIFIED +from evals.core.results import CallRecord, TaskResult from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .statistics import median @@ -16,6 +17,49 @@ "successfully does not" ) +FRICTION_SPLIT_LIMITATION = ( + "limitation: a first not_found is read as the answer to an existence question, since asking " + "has no cheaper form; only a repeat on the same tool and action is counted as friction. A " + "surface that misleads an agent into one wrong lookup is therefore not charged for it" +) + + +def split_errors(calls: list[CallRecord]) -> dict[str, int]: + """Count a row's errored calls by what kind of "no" they received. + + ``not_found`` is split in two. The first absent read of a given tool and action + is the answer to an existence question -- there is no cheaper way to ask -- and + lands in ``answered``. A second identical one means the first was not understood, + so it joins ``surface``. + """ + counts = dict.fromkeys(("navigation", "surface", "answered", "other", "unclassified"), 0) + seen_absent: set[tuple[str, str]] = set() + for call in calls: + if not call.is_error: + continue + kind = call.error_class or UNCLASSIFIED + if kind == REFUSED: + counts["navigation"] += 1 + elif kind == REJECTED: + counts["surface"] += 1 + elif kind == NOT_FOUND: + key = (call.tool, call.action or "") + if key in seen_absent: + counts["surface"] += 1 + else: + seen_absent.add(key) + counts["answered"] += 1 + elif kind == UNCLASSIFIED: + # Kept apart from `other`, which holds errors we did classify and chose + # not to charge to tool design. A row written before this field existed + # lands here in full, and a split reading zero surface friction because + # nothing was classified must not read as a surface with no friction. + counts["unclassified"] += 1 + else: + # denied/failed: real, but not attributable to tool design. + counts["other"] += 1 + return counts + @dataclass(frozen=True, slots=True) class TaskSchemaFriction: @@ -27,6 +71,11 @@ class TaskSchemaFriction: total_calls: int median_errored_calls: float errored_call_rate: float | None + navigation_calls: int = 0 + surface_calls: int = 0 + answered_calls: int = 0 + other_calls: int = 0 + unclassified_calls: int = 0 @property def address(self) -> str: @@ -42,6 +91,12 @@ class SchemaFrictionMeasurement: task_mean_errored_calls: float | None task_mean_errored_call_rate: float | None rate_task_count: int + navigation_calls: int = 0 + surface_calls: int = 0 + answered_calls: int = 0 + other_calls: int = 0 + unclassified_calls: int = 0 + total_calls: int = 0 @property def task_count(self) -> int: @@ -75,6 +130,10 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: task_rows = by_task[task_id] errored_calls = sum(row.errored_calls for row in task_rows) total_calls = sum(row.num_calls for row in task_rows) + split = {key: 0 for key in ("navigation", "surface", "answered", "other", "unclassified")} + for row in task_rows: + for key, value in split_errors(row.calls).items(): + split[key] += value tasks[task_id] = TaskSchemaFriction( task_id=task_id, repetitions=len(task_rows), @@ -82,6 +141,11 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: total_calls=total_calls, median_errored_calls=float(median([float(row.errored_calls) for row in task_rows]) or 0.0), errored_call_rate=(errored_calls / total_calls if total_calls else None), + navigation_calls=split["navigation"], + surface_calls=split["surface"], + answered_calls=split["answered"], + other_calls=split["other"], + unclassified_calls=split["unclassified"], ) absolute_values = [task.median_errored_calls for task in tasks.values()] @@ -91,9 +155,46 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: task_mean_errored_calls=(sum(absolute_values) / len(absolute_values) if absolute_values else None), task_mean_errored_call_rate=(sum(rate_values) / len(rate_values) if rate_values else None), rate_task_count=len(rate_values), + navigation_calls=sum(task.navigation_calls for task in tasks.values()), + surface_calls=sum(task.surface_calls for task in tasks.values()), + answered_calls=sum(task.answered_calls for task in tasks.values()), + other_calls=sum(task.other_calls for task in tasks.values()), + unclassified_calls=sum(task.unclassified_calls for task in tasks.values()), + total_calls=sum(task.total_calls for task in tasks.values()), ) +def _split_lines(measurement: SchemaFrictionMeasurement) -> tuple[str, ...]: + """The three numbers the single rate used to conflate.""" + total = measurement.total_calls + + def share(count: int) -> str: + return f"{count}" + (f" ({count / total:.1%})" if total else "") + + surface = measurement.surface_calls + unclassified = measurement.unclassified_calls + lines = [ + f" by kind, of {total} calls: " + f"surface friction={share(surface)}, " + f"navigation={share(measurement.navigation_calls)}, " + f"answered existence questions={share(measurement.answered_calls)}, " + f"other={share(measurement.other_calls)}, " + f"unclassified={share(unclassified)}", + ] + if unclassified: + # Never let "no surface friction" stand in for "nothing was classified". + lines.append( + f" split incomplete: {unclassified} errored call(s) carry no class — a run recorded " + "before error classes existed, or payloads the classifier does not recognise" + ) + else: + lines.append( + " surface friction is the number to act on: a well-formed call the API refused on meaning" + + ("" if surface else " — none in this run") + ) + return tuple(lines) + + def schema_friction_statement(measurement: SchemaFrictionMeasurement) -> str: """Render explicit zeros, task addresses, and the measurement boundary.""" absolute = measurement.task_mean_errored_calls @@ -108,6 +209,8 @@ def schema_friction_statement(measurement: SchemaFrictionMeasurement) -> str: f"task-mean median errored calls={absolute_text} across {measurement.task_count} tasks; " f"task-mean errored-call rate={rate_text} across {measurement.rate_task_count} tasks with calls", f" errored-call tasks: {len(flagged)}/{measurement.task_count}{flagged_text}", + *_split_lines(measurement), f" {SCHEMA_FRICTION_LIMITATION}", + f" {FRICTION_SPLIT_LIMITATION}", ) ) diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index d22cb08f..24862167 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -402,8 +402,15 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): "schema friction (same successful, trace-intact rows as call deltas): task-mean median errored calls=0.0 " "across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" " errored-call tasks: 0/1 []\n" + " by kind, of 2 calls: surface friction=0 (0.0%), navigation=0 (0.0%), " + "answered existence questions=0 (0.0%), other=0 (0.0%), unclassified=0 (0.0%)\n" + " surface friction is the number to act on: a well-formed call the API refused on " + "meaning \u2014 none in this run\n" " limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an error that is " "the correct task outcome still contributes, while calling the wrong tool successfully does not\n" + " limitation: a first not_found is read as the answer to an existence question, since asking has no " + "cheaper form; only a repeat on the same tool and action is counted as friction. A surface that " + "misleads an agent into one wrong lookup is therefore not charged for it\n" "RUN COMPLETE: 1/1 rows completed\n" "tool variability: —\n" "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py index 718f28dc..bfeec17b 100644 --- a/tests/evals/report/test_table.py +++ b/tests/evals/report/test_table.py @@ -359,8 +359,15 @@ def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): "local schema friction (same successful, trace-intact rows as call deltas): task-mean median errored " "calls=0.0 across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" "local errored-call tasks: 0/1 []\n" + "local by kind, of 2 calls: surface friction=0 (0.0%), navigation=0 (0.0%), answered " + "existence questions=0 (0.0%), other=0 (0.0%), unclassified=0 (0.0%)\n" + "local surface friction is the number to act on: a well-formed call the API refused on " + "meaning \u2014 none in this run\n" "local limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an " "error that is the correct task outcome still contributes, while calling the wrong tool successfully does not\n" + "local limitation: a first not_found is read as the answer to an existence question, since " + "asking has no cheaper form; only a repeat on the same tool and action is counted as friction. A " + "surface that misleads an agent into one wrong lookup is therefore not charged for it\n" "local RUN COMPLETE: 1/1 rows completed\n" ) diff --git a/tests/evals/test_error_class.py b/tests/evals/test_error_class.py new file mode 100644 index 00000000..3a42a40b --- /dev/null +++ b/tests/evals/test_error_class.py @@ -0,0 +1,218 @@ +"""Classifying what kind of "no" a tool call received, and what counts as friction. + +Every payload below is one that a real battery produced. One errored-call count +answered three unrelated questions at once: on an unpatched 28-tool surface, 31 of +40 errors were the schema correcting a malformed call, one was a genuine tool-design +defect, and one was a fair question fairly answered. Reading them as one number is +what made the defect invisible. +""" + +from __future__ import annotations + +import pytest + +from evals.core.error_class import ( + DENIED, + FAILED, + NOT_FOUND, + REFUSED, + REJECTED, + UNCLASSIFIED, + classify_error, +) +from evals.core.results import CallRecord +from evals.report.schema_friction import split_errors + +# --- the classifier, on payloads observed in real runs ----------------------- + +OBSERVED = [ + # This server's own refusals: no status, no pydantic shape, deliberate wording. + ("Error: project requires an action. It takes: archive, create, delete, list.", REFUSED), + ("Error: action 'create' does not take: points. It takes: description, name.", REFUSED), + # FastMCP rejecting a call against the tool signature, before the body runs. + ( + "1 validation error for call[project]\naction\n Missing required argument " + "[type=missing_argument, input_value={}, input_type=dict]", + REFUSED, + ), + ( + "1 validation error for call[workspace]\naction\n Input should be 'get_features' " + "or 'update_features' [type=literal_error, input_value='list', input_type=str]", + REFUSED, + ), + # The API answering a fair existence question. + ("Error calling tool 'project_estimate': HTTP 404: Not Found: Estimate not found", NOT_FOUND), + # The API refusing the meaning of a well-formed call -- the defect class. + ("HTTP 400: Bad Request: The old cycle is not completed yet", REJECTED), + ("HTTP 409: Conflict: name: The project name is already taken", REJECTED), + # Plan and permission gates say nothing about the tool surface. + ("HTTP 402: Payment Required: Upgrade your plan to access Initiatives", DENIED), + ("HTTP 403: Forbidden: Customer feature is not enabled for this workspace", DENIED), + ("HTTP 500: Internal Server Error", FAILED), +] + + +@pytest.mark.parametrize(("payload", "expected"), OBSERVED, ids=[e + ":" + p[:28] for p, e in OBSERVED]) +def test_an_observed_payload_lands_in_its_class(payload: str, expected: str): + assert classify_error(payload) == expected + + +def test_a_status_outranks_wording(): + """A 404 that happens to mention an argument is still an absent resource. + + Only a payload with no status at all can be a schema refusal, because that is + exactly the case where the call never reached the API. + """ + assert classify_error("HTTP 404: Not Found: missing required argument foo") == NOT_FOUND + + +def test_an_unrecognised_payload_is_never_guessed_into_a_class(): + """Silently sorting the unknown into `refused` would inflate the one number + that is supposed to be attributable to our own schema.""" + assert classify_error("something went sideways") == UNCLASSIFIED + assert classify_error("") == UNCLASSIFIED + assert classify_error(None) == UNCLASSIFIED + + +def test_a_foreign_surface_still_classifies_by_status(): + """The battery scores servers it has never seen -- a 177-tool build, a future v2. + + Those emit neither this server's refusal wording nor its tool names, so status + has to carry them. Coupling the classifier to `ACTIONS` would end that. + """ + assert classify_error("HTTP 400: Bad Request: whatever a foreign server says") == REJECTED + assert classify_error("HTTP 404: Not Found") == NOT_FOUND + + +# --- the split, including the rule that keeps a fair question out of friction --- + + +def err(tool: str, action: str | None, kind: str) -> CallRecord: + return CallRecord(tool=tool, action=action, is_error=True, error_class=kind) + + +def test_an_unclassified_error_is_not_filed_beside_ones_we_chose_not_to_charge(): + """`other` means classified and deliberately not charged to tool design. + `unclassified` means we do not know, which a reader must be able to tell apart. + """ + counts = split_errors([err("project", "list", DENIED), err("project", "list", UNCLASSIFIED)]) + assert counts["other"] == 1 + assert counts["unclassified"] == 1 + + +def test_each_kind_lands_in_its_own_column(): + counts = split_errors( + [ + err("project", None, REFUSED), + err("cycle", "transfer_workitems", REJECTED), + err("initiative", "list", DENIED), + CallRecord(tool="project", action="list"), # a success is not counted anywhere + ] + ) + assert counts == {"navigation": 1, "surface": 1, "answered": 0, "other": 1, "unclassified": 0} + + +def test_a_first_absent_read_is_an_answer_not_friction(): + """`project_estimate retrieve` -> 404 before creating one is the correct move. + + Three separate models made this exact call and each was charged for it. There is + no cheaper way to ask whether something exists than to ask. + """ + counts = split_errors([err("project_estimate", "retrieve", NOT_FOUND)]) + assert counts["answered"] == 1 + assert counts["surface"] == 0 + + +def test_a_repeated_absent_read_is_friction(): + """Asking twice means the first answer did not land, which is the surface's problem.""" + counts = split_errors( + [ + err("project_estimate", "retrieve", NOT_FOUND), + err("project_estimate", "retrieve", NOT_FOUND), + err("project_estimate", "retrieve", NOT_FOUND), + ] + ) + assert counts["answered"] == 1 + assert counts["surface"] == 2 + + +def test_absent_reads_of_different_things_are_each_their_own_question(): + counts = split_errors( + [ + err("project_estimate", "retrieve", NOT_FOUND), + err("cycle", "retrieve", NOT_FOUND), + err("project_estimate", "list_points", NOT_FOUND), + ] + ) + assert counts["answered"] == 3 + assert counts["surface"] == 0 + + +def test_an_unclassified_error_is_never_counted_as_surface_friction(): + """Surface friction is the number someone will act on, so it may only hold + calls we can actually attribute to tool design.""" + counts = split_errors([err("project", "list", UNCLASSIFIED), err("project", "list", FAILED)]) + assert counts["surface"] == 0 + assert counts["unclassified"] == 1 + assert counts["other"] == 1 + + +def test_a_row_from_before_this_field_existed_does_not_crash_or_inflate(): + """Older result files carry is_error with no error_class.""" + counts = split_errors([CallRecord(tool="project", action="list", is_error=True)]) + assert counts["unclassified"] == 1 + assert counts["surface"] == 0 + assert counts["other"] == 0 + + +# --- the whole path, because a key dropped anywhere on it reaches no report ---- + + +def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): + """The classifier ran and the report still said "unclassified", because the + sidecar reader rebuilds each call from an explicit key list and did not copy the + field. Every hop is asserted here: proxy row -> sidecar reader -> AgentRun -> + TaskResult -> serialized row -> reloaded row. + """ + import json + + from evals.core.results import AgentRun, TaskResult, Usage, agent_run_to_task_result + from evals.drivers.cli.sidecar import load_proxy_sidecar + + sidecar = tmp_path / "proxy-sidecar.jsonl" + rows = [ + { + "tool": "cycle", + "args": {"action": "transfer_workitems"}, + "is_error": True, + "error_class": REJECTED, + "result_chars": 145, + "duration_ms": 115, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + sidecar.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + calls, _status = load_proxy_sidecar(sidecar) + assert calls and calls[0].get("error_class") == REJECTED, "the sidecar reader dropped it" + + agent = agent_run_to_task_result( + AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn") + ) + assert agent.calls[0].error_class == REJECTED, "agent_run_to_task_result dropped it" + + serialized = json.loads(json.dumps(agent.to_row())) + assert serialized["calls"][0]["error_class"] == REJECTED, "serialization dropped it" + + reloaded = TaskResult.from_row(serialized) + assert reloaded.calls[0].error_class == REJECTED, "reload dropped it" + assert split_errors(reloaded.calls)["surface"] == 1, "the report did not see it" From ce2158a1ceac9fb21dda2992667dbb320adbaa99 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 20 Aug 2026 15:41:26 +0530 Subject: [PATCH 73/93] Document the errored-call split in DESIGN and README --- evals/DESIGN.md | 19 +++++++++++++++++++ evals/README.md | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index df3fa679..1716318b 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -81,6 +81,25 @@ zero-attempt task has an undefined rate rather than an invented zero rate. `is_e MCP-level error flag: it counts all tool-reported failures, including an error that is the correct outcome, and cannot detect an agent that successfully calls the wrong tool. +That single count answers three unrelated questions at once, so errored calls are also split by +the kind of refusal they received, classified in the proxy where the payload still exists and +stored as a category rather than text: + +| reported as | from | means | +|---|---|---| +| navigation | `refused` | turned away without acting — a missing field, a stray argument, a value outside an enum. A property of the tool schema; the API was never asked. | +| surface friction | `rejected` | well formed, and the API refused its meaning. Undocumented preconditions live here. The number to act on. | +| answered existence question | first `not_found` per tool and action | an absent read is the answer, not an obstacle; asking has no cheaper form. A *repeat* is charged to surface friction, because the first answer did not land. | +| other | `denied`, `failed` | credentials, plan, or a broken server. Real, but not attributable to tool design. | +| unclassified | everything else | reported apart from `other` on purpose: a split reading zero surface friction because nothing was classified must not be mistaken for a surface with no friction. Rows written before this field existed land here in full. | + +Classification reads HTTP status and the FastMCP/Pydantic validation shape, never this server's +`ACTIONS` table, so a foreign surface still classifies by status — the same battery has scored +both a 28-tool and a 177-tool build. Two patterns matching this server's own refusal wording are +additive. Status outranks wording: a 404 whose body mentions a missing argument is still an +absent resource, since only a payload with no status at all can be a call that never reached the +API. The split cannot charge a surface for misleading an agent into a single wrong lookup. + Single-run success headlines use the same sampling unit: each evaluated task contributes its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. The pooled repetition rate and its Wilson interval remain visible as a descriptive figure, diff --git a/evals/README.md b/evals/README.md index 6cfce5a0..dc7444e7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -172,6 +172,14 @@ deltas in A/B output and task IDs for investigation. This is a proxy, not a pure counter: MCP `is_error` also marks correct expected failures, while a successful call to the wrong tool is invisible to it. +Because one count conflates three different things, the same reports break errored calls into +**surface friction** (a well-formed call the API refused on meaning — the number to act on), +**navigation** (the schema correcting a malformed call), **answered existence questions** (a +first absent read, which is an answer rather than an obstacle), plus `other` and +`unclassified`. A non-zero `unclassified` prints a "split incomplete" line, so zero surface +friction never stands in for nothing having been classified — including when reading a result +file recorded before the split existed. See DESIGN.md for the classification rules. + Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and fixture names. The battery contains exactly what the agent is asked and no expectation about From 85708d94c5d70ced0c6748acb66bd5eaabd1770f Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 20 Aug 2026 17:03:22 +0530 Subject: [PATCH 74/93] Count a refusal the server reports as a successful result This server answers a malformed call with a plain result whose text begins "Error: ", so the protocol reports success. About 47 refusals per 35-task battery arrived that way and were counted as successes -- a measured 12.8% errored-call rate against a true refusal rate near 28%. Making the server flag them instead cost +13% total calls, a median of one extra call per task, so that change was reverted. The visibility problem is the harness's to solve, and it belongs here: the proxy now classifies a refusal it recognises in a successful payload, and the split counts it while naming it apart, because it is absent from `errored_calls` -- the protocol-flag total the rest of the report and every earlier run use. Detection is deliberately narrow: only wording this server owns, and the stray-argument form must carry both of its halves, so a tool result that happens to quote one phrase is not miscounted. A foreign surface contributes nothing here and still classifies its flagged errors by status. Unit-tested against real payload shapes including two near-misses. Not yet observed live: the smoke run that would have exercised it made no refused call, so this path has no live confirmation behind it yet. --- evals/core/error_class.py | 21 ++++++++++++++ evals/proxy.py | 13 +++++---- evals/report/schema_friction.py | 20 +++++++++++-- tests/evals/test_error_class.py | 51 ++++++++++++++++++++++++++++++--- 4 files changed, 93 insertions(+), 12 deletions(-) diff --git a/evals/core/error_class.py b/evals/core/error_class.py index e556adda..85098587 100644 --- a/evals/core/error_class.py +++ b/evals/core/error_class.py @@ -73,6 +73,26 @@ } +def detect_refusal(payload: str | None) -> str | None: + """Return ``REFUSED`` for a refusal that arrived flagged as a *successful* result. + + This server answers a malformed call with a plain result whose text begins + "Error: ", so the protocol reports success and a caller counting failures sees + none -- about 47 per 35-task battery. Classifying those anyway keeps the metric + honest without asking the server to change what every agent receives. + + Deliberately narrow. Only wording this server owns counts, and the stray-argument + form must carry both of its halves, so an ordinary tool result that happens to + quote one phrase is not miscounted as a refusal. + """ + text = (payload or "").lower() + if "requires an action. it takes:" in text: + return REFUSED + if "does not take:" in text and "it takes:" in text: + return REFUSED + return None + + def classify_error(payload: str | None) -> str: """Return the category of a failed call from its error payload. @@ -105,6 +125,7 @@ def classify_error(payload: str | None) -> str: __all__ = [ "ERROR_CLASSES", + "detect_refusal", "NAVIGATION_CLASSES", "SURFACE_FRICTION_CLASSES", "classify_error", diff --git a/evals/proxy.py b/evals/proxy.py index e0d85bc8..0e32e10f 100644 --- a/evals/proxy.py +++ b/evals/proxy.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any -from evals.core.error_class import classify_error +from evals.core.error_class import classify_error, detect_refusal from evals.core.evidence import ( EVIDENCE_SENTINELS_ENV, consume_evidence_config, @@ -328,10 +328,13 @@ def on_server_message(self, obj: dict[str, Any]) -> None: "duration_ms": duration_ms, "seq": pending["seq"], } - if is_error: - # Classified here because this is the last place the payload exists: - # rows keep result_chars, not the text. Only the category is stored. - row["error_class"] = classify_error(result_text) + # Classified here because this is the last place the payload exists: rows keep + # result_chars, not the text. Only the category is stored. A refusal the server + # reports as a successful result is classified too, so the metric is not blind + # to it -- see detect_refusal. + error_class = classify_error(result_text) if is_error else detect_refusal(result_text) + if error_class is not None: + row["error_class"] = error_class if self.evidence_active: # Persist only labels matched from non-enumerable sentinels and # target-bound aggregate values the agent already received. The diff --git a/evals/report/schema_friction.py b/evals/report/schema_friction.py index 853ccc3e..25ceefa9 100644 --- a/evals/report/schema_friction.py +++ b/evals/report/schema_friction.py @@ -32,11 +32,16 @@ def split_errors(calls: list[CallRecord]) -> dict[str, int]: lands in ``answered``. A second identical one means the first was not understood, so it joins ``surface``. """ - counts = dict.fromkeys(("navigation", "surface", "answered", "other", "unclassified"), 0) + counts = dict.fromkeys(("navigation", "surface", "answered", "other", "unclassified", "unflagged"), 0) seen_absent: set[tuple[str, str]] = set() for call in calls: - if not call.is_error: + if not call.is_error and call.error_class is None: continue + if not call.is_error: + # A refusal the server reported as a successful result. It is counted here + # and named separately, because it is absent from `errored_calls` -- the + # protocol-flag total the rest of the report and every earlier run use. + counts["unflagged"] += 1 kind = call.error_class or UNCLASSIFIED if kind == REFUSED: counts["navigation"] += 1 @@ -76,6 +81,7 @@ class TaskSchemaFriction: answered_calls: int = 0 other_calls: int = 0 unclassified_calls: int = 0 + unflagged_refusals: int = 0 @property def address(self) -> str: @@ -96,6 +102,7 @@ class SchemaFrictionMeasurement: answered_calls: int = 0 other_calls: int = 0 unclassified_calls: int = 0 + unflagged_refusals: int = 0 total_calls: int = 0 @property @@ -130,7 +137,7 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: task_rows = by_task[task_id] errored_calls = sum(row.errored_calls for row in task_rows) total_calls = sum(row.num_calls for row in task_rows) - split = {key: 0 for key in ("navigation", "surface", "answered", "other", "unclassified")} + split = {key: 0 for key in ("navigation", "surface", "answered", "other", "unclassified", "unflagged")} for row in task_rows: for key, value in split_errors(row.calls).items(): split[key] += value @@ -146,6 +153,7 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: answered_calls=split["answered"], other_calls=split["other"], unclassified_calls=split["unclassified"], + unflagged_refusals=split["unflagged"], ) absolute_values = [task.median_errored_calls for task in tasks.values()] @@ -160,6 +168,7 @@ def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: answered_calls=sum(task.answered_calls for task in tasks.values()), other_calls=sum(task.other_calls for task in tasks.values()), unclassified_calls=sum(task.unclassified_calls for task in tasks.values()), + unflagged_refusals=sum(task.unflagged_refusals for task in tasks.values()), total_calls=sum(task.total_calls for task in tasks.values()), ) @@ -181,6 +190,11 @@ def share(count: int) -> str: f"other={share(measurement.other_calls)}, " f"unclassified={share(unclassified)}", ] + if measurement.unflagged_refusals: + lines.append( + f" {measurement.unflagged_refusals} refusal(s) arrived flagged as successful results, so they " + "are counted above but not in the errored-call total" + ) if unclassified: # Never let "no surface friction" stand in for "nothing was classified". lines.append( diff --git a/tests/evals/test_error_class.py b/tests/evals/test_error_class.py index 3a42a40b..ce56c071 100644 --- a/tests/evals/test_error_class.py +++ b/tests/evals/test_error_class.py @@ -19,6 +19,7 @@ REJECTED, UNCLASSIFIED, classify_error, + detect_refusal, ) from evals.core.results import CallRecord from evals.report.schema_friction import split_errors @@ -109,7 +110,14 @@ def test_each_kind_lands_in_its_own_column(): CallRecord(tool="project", action="list"), # a success is not counted anywhere ] ) - assert counts == {"navigation": 1, "surface": 1, "answered": 0, "other": 1, "unclassified": 0} + assert counts == { + "navigation": 1, + "surface": 1, + "answered": 0, + "other": 1, + "unclassified": 0, + "unflagged": 0, + } def test_a_first_absent_read_is_an_answer_not_friction(): @@ -205,9 +213,7 @@ def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): calls, _status = load_proxy_sidecar(sidecar) assert calls and calls[0].get("error_class") == REJECTED, "the sidecar reader dropped it" - agent = agent_run_to_task_result( - AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn") - ) + agent = agent_run_to_task_result(AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn")) assert agent.calls[0].error_class == REJECTED, "agent_run_to_task_result dropped it" serialized = json.loads(json.dumps(agent.to_row())) @@ -216,3 +222,40 @@ def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): reloaded = TaskResult.from_row(serialized) assert reloaded.calls[0].error_class == REJECTED, "reload dropped it" assert split_errors(reloaded.calls)["surface"] == 1, "the report did not see it" + + +def test_a_refusal_the_server_called_a_success_is_still_counted(): + """The server answers a malformed call with a plain result whose text says + "Error", so the protocol reports success. Roughly 47 per battery arrived that + way and were counted as successes. They are counted here and named apart, + because they are absent from the errored-call total every earlier run used. + """ + counts = split_errors( + [ + CallRecord(tool="project", action=None, is_error=False, error_class=REFUSED), + CallRecord(tool="project", action="list", is_error=True, error_class=REFUSED), + ] + ) + assert counts["navigation"] == 2 + assert counts["unflagged"] == 1 + + +def test_a_plain_successful_call_is_still_not_counted(): + counts = split_errors([CallRecord(tool="project", action="list")]) + assert sum(counts.values()) == 0 + + +REFUSAL_TEXTS = [ + ('{"content":[{"type":"text","text":"Error: project requires an action. It takes: list."}]}', REFUSED), + ('{"content":[{"type":"text","text":"Error: action \'create\' does not take: points. It takes: name."}]}', REFUSED), + # An ordinary result quoting one half of the stray-argument wording is not a refusal. + ('{"content":[{"type":"text","text":"the docs say the endpoint does not take: a body"}]}', None), + ('{"content":[{"type":"text","text":"[{"id":"abc","name":"Delivery Planning"}]"}]}', None), +] + + +@pytest.mark.parametrize( + ("payload", "expected"), REFUSAL_TEXTS, ids=["missing-action", "stray-arg", "quotes-one-half", "ordinary-result"] +) +def test_only_this_servers_own_refusal_wording_is_detected(payload: str, expected: str | None): + assert detect_refusal(payload) == expected From 86465816309bbf503392bfbc20c23973f16036ff Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 20 Aug 2026 19:33:59 +0530 Subject: [PATCH 75/93] Survive a project-name collision, and stop reporting a dirty workspace as clean Two defects found while checking whether the two eval workspaces were equivalent enough to run in parallel. Both were live, and the second one had already cost a measurement. A project-name collision was unrecoverable. Plane refuses a duplicate project name with the same 409 and the same collision wording it uses for a duplicate identifier -- `name: The project name is already taken` against `identifier: ...` -- so is_identifier_collision could not tell them apart and matched both. A name collision therefore entered the retry loop and spent all eight attempts regenerating an identifier suffix that was never the problem, then re-raised. Names come from a two-title pool and 64 words, so residue from a crashed run makes this a matter of when: it is what failed L3 in the 177-tool arm on 2026-08-19, five minutes after that same arm leaked the project it then collided with. So the arm's 67/70 is really 68/70 with one seed failure caused by workspace residue rather than by the surface. Now the two are distinguished by which field the body names, and a name collision advances to the next name instead. eval_project_name_variants yields the existing deterministic name first -- so a resumed run and its teardown still agree on it -- then the rest of the word pool in a reproducible order. Both call sites record the name that was actually created: the agent is told this name in its preamble, so a retry that was not written back would name a project that does not exist. A body naming both fields is still treated as the identifier case, which is what this did before names could be retried at all. Cleanup could not delete a leftover workspace-level Bug type. Only Incident was a deletion target; Bug appeared solely to locate the Severity property to remove. A workspace holding Bug types therefore printed `sentinel_matches=0` and `nothing to delete` -- it reported clean while holding types that skew any task reading the workspace-level list, which is exactly how I read one of these workspaces as clean earlier today. Cleanup now matches every fixture name, duplicates of one name included, and reports any workspace-level type the harness never creates instead of passing over it in silence. Those are deleted only with --unowned: this runs against instances it does not own, so a type it did not create is reported, not removed. Verified: 686 offline tests pass, ruff clean. Each new test was confirmed to fail against the pre-fix behaviour -- name-vs-identifier classification, all three retry paths, and both cleanup paths -- rather than only passing against the new one. Not fixed here: teardown leaked two projects during those arms while every result row recorded cleanup_error 0, so it reported success and left them behind. The name-retry above removes the consequence, not the leak. Diagnosing why the id never reached teardown is separate work. Co-Authored-By: Claude Opus 5 (1M context) --- evals/cleanup.py | 60 ++++++++++- evals/core/fixtures.py | 34 +++++- evals/seed/__init__.py | 6 +- evals/seed/build.py | 16 ++- evals/seed/item_types.py | 3 + evals/seed/projects.py | 75 +++++++++---- tests/evals/seed/test_seed.py | 195 ++++++++++++++++++++++++++++++++-- 7 files changed, 345 insertions(+), 44 deletions(-) diff --git a/evals/cleanup.py b/evals/cleanup.py index e079fc64..712e8b17 100644 --- a/evals/cleanup.py +++ b/evals/cleanup.py @@ -18,7 +18,7 @@ ) from evals.seed.item_types import ( BUG_TYPE_NAME, - INCIDENT_TYPE_NAME, + FIXTURE_WORK_ITEM_TYPE_NAMES, is_severity_property, is_work_item_type_named, list_workspace_properties_for_type, @@ -113,8 +113,19 @@ def list_sentinel_workspace_artifacts(plane: Any, workspace_slug: str) -> list[d if callable(getattr(type_api, "list", None)): for row in list_workspace_work_item_types(plane, workspace_slug): object_id = getattr(row, "id", None) - if object_id is not None and is_work_item_type_named(row, INCIDENT_TYPE_NAME): - artifacts.append({"kind": "work_item_type", "id": object_id, "name": INCIDENT_TYPE_NAME}) + if object_id is None: + continue + # Every fixture name, not just Incident. Bug used to be reachable only as the type + # whose Severity property gets removed, so leftover Bug types were both undeletable + # by this tool and counted as "nothing to delete" -- a workspace reported clean while + # holding types that skew any task reading the workspace-level list. Duplicates of + # one name each match, so a double-seeded type is fully removed. + matched = next( + (name for name in FIXTURE_WORK_ITEM_TYPE_NAMES if is_work_item_type_named(row, name)), + None, + ) + if matched is not None: + artifacts.append({"kind": "work_item_type", "id": object_id, "name": matched}) property_api = getattr(plane, "workspace_work_item_properties", None) links_api = getattr(type_api, "properties", None) @@ -127,6 +138,28 @@ def list_sentinel_workspace_artifacts(plane: Any, workspace_slug: str) -> list[d return artifacts +def list_unowned_workspace_work_item_types(plane: Any, workspace_slug: str) -> list[dict[str, Any]]: + """Return workspace-level work item types this harness never creates. + + Reported rather than deleted by default: a type the harness did not create may be a real + workspace's configuration, and this runs against instances it does not own. They still + have to be *visible*, because a workspace holding types another workspace lacks skews + every task that reads the workspace-level list, and silence there reads as clean. + """ + type_api = getattr(plane, "workspace_work_item_types", None) + if not callable(getattr(type_api, "list", None)): + return [] + unowned: list[dict[str, Any]] = [] + for row in list_workspace_work_item_types(plane, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is None: + continue + if any(is_work_item_type_named(row, name) for name in FIXTURE_WORK_ITEM_TYPE_NAMES): + continue + unowned.append({"kind": "work_item_type", "id": object_id, "name": (getattr(row, "name", "") or "").strip()}) + return unowned + + def _sentinel_description(artifact: dict[str, Any]) -> str: kind = str(artifact["kind"]).replace("_", " ") return f"{kind} {artifact['name']!r} ({artifact['id']})" @@ -168,9 +201,18 @@ def delete_sentinel_workspace_artifacts( return deleted, failed -def _cleanup_sentinels(plane: Any, workspace_slug: str, *, yes: bool) -> int: +def _cleanup_sentinels(plane: Any, workspace_slug: str, *, yes: bool, unowned: bool = False) -> int: artifacts = list_sentinel_workspace_artifacts(plane, workspace_slug) + others = list_unowned_workspace_work_item_types(plane, workspace_slug) + if unowned: + artifacts = artifacts + others print(f"workspace={workspace_slug} sentinel_matches={len(artifacts)}") + if others and not unowned: + # Never let a zero match count imply a clean workspace while these sit here. + print(f"note: {len(others)} workspace work item type(s) present that this tool did not create:") + for artifact in others: + print(f" {_sentinel_description(artifact)}") + print(" add --unowned to delete them too") if not artifacts: print("nothing to delete") return 0 @@ -192,6 +234,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Clean fixed-name workspace sentinels instead of projects", ) + p.add_argument( + "--unowned", + action="store_true", + help="With --sentinels, also delete workspace work item types this harness never creates", + ) p.add_argument( "--yes", action="store_true", @@ -208,7 +255,10 @@ def main(argv: list[str] | None = None) -> int: return 2 if args.sentinels: - return _cleanup_sentinels(plane, workspace_slug, yes=args.yes) + return _cleanup_sentinels(plane, workspace_slug, yes=args.yes, unowned=args.unowned) + if args.unowned: + print("error: --unowned only applies with --sentinels", file=sys.stderr) + return 2 projects = list_projects_with_prefix(plane, workspace_slug, args.prefix) print(f"workspace={workspace_slug} prefix={args.prefix!r} matches={len(projects)}") diff --git a/evals/core/fixtures.py b/evals/core/fixtures.py index 9050ccf4..e43100e5 100644 --- a/evals/core/fixtures.py +++ b/evals/core/fixtures.py @@ -6,6 +6,8 @@ from __future__ import annotations +from collections.abc import Iterator + CUSTOMER_NAME = "Acme Corp" CUSTOMER_REQUEST_NAME = "SSO support" EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" @@ -169,26 +171,48 @@ def is_evaluation_customer_name(name: str | None) -> bool: ) +def _suffix_word_index(run_prefix: str) -> int: + """Map a run prefix onto ``PROJECT_SUFFIX_WORDS``, tolerating non-hex prefixes.""" + try: + return int(str(run_prefix)[:8], 16) + except ValueError: + return sum(ord(ch) for ch in str(run_prefix)) + + def eval_project_name(run_prefix: str, *, second: bool = False) -> str: """Build a seeded project's display name: readable, and never id-shaped. Deterministic in ``run_prefix`` so the same run always produces the same name, which keeps a resumed run and its teardown in agreement. """ - try: - index = int(str(run_prefix)[:8], 16) - except ValueError: - index = sum(ord(ch) for ch in str(run_prefix)) - word = PROJECT_SUFFIX_WORDS[index % len(PROJECT_SUFFIX_WORDS)] + word = PROJECT_SUFFIX_WORDS[_suffix_word_index(run_prefix) % len(PROJECT_SUFFIX_WORDS)] title = PROJECT_TITLES[1 if second else 0] return f"{EVAL_PROJECT_PREFIX}{title} {word}" +def eval_project_name_variants(run_prefix: str, *, second: bool = False) -> Iterator[str]: + """Yield the deterministic name, then every other word in order. + + The first name is exactly ``eval_project_name(run_prefix, second=second)``, so a + resumed run and its teardown still agree on it. The rest exist only so a leftover + project from a crashed run cannot fail a fresh one: Plane rejects a duplicate project + name with a 409, and the word pool is small enough that residue makes that collision + a matter of when. Walking forward from the deterministic index keeps the fallback + order reproducible too. + """ + words = PROJECT_SUFFIX_WORDS + start = _suffix_word_index(run_prefix) % len(words) + title = PROJECT_TITLES[1 if second else 0] + for offset in range(len(words)): + yield f"{EVAL_PROJECT_PREFIX}{title} {words[(start + offset) % len(words)]}" + + __all__ = [ "EVAL_PROJECT_PREFIX", "PROJECT_SUFFIX_WORDS", "PROJECT_TITLES", "eval_project_name", + "eval_project_name_variants", "BLOCKING_REFERENCE_ADDRESS", "BLOCKING_SOURCE_TITLE", "BLOCKING_TARGET_TITLE", diff --git a/evals/seed/__init__.py b/evals/seed/__init__.py index a7c13acf..99d30604 100644 --- a/evals/seed/__init__.py +++ b/evals/seed/__init__.py @@ -28,10 +28,11 @@ MAIN_PROJECT_BUG_TITLES, PLANE_PROJECT_IDENTIFIER_MAX_LENGTH, SECOND_PROJECT_BUG_TITLES, - create_project_with_identifier_retry, + create_project_with_collision_retry, enable_project_features, enable_workspace_features, is_identifier_collision, + is_name_collision, secrets, seed_second_project, ) @@ -158,11 +159,12 @@ "_gate_activity_worker", "check_workspace_fixture_collisions", "collision_categories", - "create_project_with_identifier_retry", + "create_project_with_collision_retry", "enable_project_features", "enable_workspace_features", "find_completed_state", "is_identifier_collision", + "is_name_collision", "is_evaluation_customer_name", "is_plan_gate", "plan_gate_skips", diff --git a/evals/seed/build.py b/evals/seed/build.py index 09caf5b7..98726c14 100644 --- a/evals/seed/build.py +++ b/evals/seed/build.py @@ -8,7 +8,7 @@ from plane import PlaneClient from evals.core.errors import TaskSkipped -from evals.core.fixtures import eval_project_name +from evals.core.fixtures import eval_project_name_variants from .customers import ( CUSTOMER_NAME, @@ -33,7 +33,7 @@ from .labels import seed_labels from .modules import seed_module from .projects import ( - create_project_with_identifier_retry, + create_project_with_collision_retry, enable_project_features, enable_workspace_features, seed_second_project, @@ -271,7 +271,8 @@ def seed( even if a later fixture step raises (F5). """ run_prefix = run_id[:8] - project_name = eval_project_name(run_prefix) + name_variants = eval_project_name_variants(run_prefix) + project_name = next(name_variants) workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] # Reset known keys while preserving object identity for the caller. @@ -327,17 +328,22 @@ def seed( task_collision_categories = collision_categories(needs, task_id) check_workspace_fixture_collisions(plane, workspace_slug, task_collision_categories) - # EV + 8 hex chars; retry with a new suffix on soft-delete identifier collisions. - project = create_project_with_identifier_retry( + # EV + 8 hex chars; a new suffix on soft-delete identifier collisions, the next name + # variant if the name itself is taken by residue from a crashed run. + project = create_project_with_collision_retry( plane, workspace_slug, name=project_name, identifier_prefix="EV", initial_suffix=run_prefix.upper(), + name_variants=name_variants, ) ctx["project_id"] = project.id record_seeded_entity(ctx, "project", project.id) ctx["project_identifier"] = getattr(project, "identifier", None) + # The created name, not the requested one: the agent is told this in its preamble, so a + # name-collision retry that was not written back would name a project that does not exist. + ctx["project_name"] = getattr(project, "name", None) or project_name # Workspace first, then project. Seeding is per task-rep, so S5 turning workspace # customers off must be undone in teardown or a later C1 rep 403s. diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py index 3bab4719..98e928e6 100644 --- a/evals/seed/item_types.py +++ b/evals/seed/item_types.py @@ -12,6 +12,9 @@ BUG_TYPE_NAME = "Bug" INCIDENT_TYPE_NAME = "Incident" +# Every workspace-level work item type name this harness creates. Cleanup deletes these and +# reports anything else it finds, because the harness runs against an instance it does not own. +FIXTURE_WORK_ITEM_TYPE_NAMES = (BUG_TYPE_NAME, INCIDENT_TYPE_NAME) SEVERITY_PROPERTY_NAME = "Severity" diff --git a/evals/seed/projects.py b/evals/seed/projects.py index 7160977c..e9fc927f 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -3,6 +3,7 @@ from __future__ import annotations import secrets +from collections.abc import Iterator from typing import Any from plane import PlaneClient @@ -12,7 +13,7 @@ from plane.models.workspaces import WorkspaceFeature from evals.core.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence -from evals.core.fixtures import eval_project_name +from evals.core.fixtures import eval_project_name_variants from .gates import is_plan_gate from .identities import record_seeded_entity @@ -35,12 +36,8 @@ ) -def is_identifier_collision(exc: BaseException) -> bool: - """True when project create failed because the identifier is already taken. - - Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare - ``identifier`` mention (validation shape errors) must not trigger retry. - """ +def _is_collision(exc: BaseException) -> bool: + """True for an HTTP 400/409 whose body reads as a uniqueness conflict.""" if not isinstance(exc, HttpError): return False if exc.status_code not in (400, 409): @@ -49,19 +46,50 @@ def is_identifier_collision(exc: BaseException) -> bool: return any(keyword in blob for keyword in ("already", "exists", "taken")) -def create_project_with_identifier_retry( +def is_name_collision(exc: BaseException) -> bool: + """True when project create failed because the project *name* is already taken. + + Plane answers both conflicts with the same 409 and the same collision wording -- + ``name: The project name is already taken`` against + ``identifier: ...`` -- so the named field is the only thing separating them. A body + mentioning the identifier is treated as an identifier collision, because retrying a + fresh suffix is cheap and was the behaviour before names could be retried at all. + """ + if not _is_collision(exc): + return False + blob = f"{exc} {exc.response!s}".lower() + return "name" in blob and "identifier" not in blob + + +def is_identifier_collision(exc: BaseException) -> bool: + """True when project create failed because the identifier is already taken. + + Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare + ``identifier`` mention (validation shape errors) must not trigger retry. + """ + return _is_collision(exc) and not is_name_collision(exc) + + +def create_project_with_collision_retry( plane: PlaneClient, workspace_slug: str, *, name: str, identifier_prefix: str, initial_suffix: str, + name_variants: Iterator[str] | None = None, ) -> Any: - """Create a project, regenerating the identifier suffix on soft-delete collisions. + """Create a project, retrying past both kinds of uniqueness conflict. + + Plane soft-deletes reserve identifiers, so an identifier collision draws a new random + 8-char hex suffix. A *name* collision means a project of that name already exists -- + residue from a crashed run, since teardown deletes by recorded id -- and the suffix was + never the problem, so it advances to the next name from ``name_variants`` instead. + Without ``name_variants`` a name collision raises immediately rather than burning the + budget regenerating an identifier that was already fine. - Plane soft-deletes reserve identifiers; a 409 (or identifier-in-message error) - triggers a new random 8-char hex suffix. At most eight attempts are made, - then the last collision error is raised again. + Both kinds share the ``PROJECT_CREATE_ATTEMPT_LIMIT`` budget, so this survives up to + seven collisions of either kind in one create; past that the last error is re-raised. """ if len(identifier_prefix) + PROJECT_IDENTIFIER_SUFFIX_LENGTH > PLANE_PROJECT_IDENTIFIER_MAX_LENGTH: raise ValueError( @@ -73,9 +101,7 @@ def create_project_with_identifier_retry( if len(suffix) < PROJECT_IDENTIFIER_SUFFIX_LENGTH: suffix = (suffix + secrets.token_hex(4).upper())[:PROJECT_IDENTIFIER_SUFFIX_LENGTH] last_exc: BaseException | None = None - for attempt in range(PROJECT_CREATE_ATTEMPT_LIMIT): - if attempt > 0: - suffix = secrets.token_hex(4).upper() # 8 hex chars / 32 bits + for _attempt in range(PROJECT_CREATE_ATTEMPT_LIMIT): identifier = f"{identifier_prefix}{suffix}" try: return plane.projects.create( @@ -83,13 +109,21 @@ def create_project_with_identifier_retry( data=CreateProject(name=name, identifier=identifier), ) except Exception as exc: + if is_name_collision(exc): + next_name = next(name_variants, None) if name_variants is not None else None + if next_name is None: + raise + name = next_name + last_exc = exc + continue if is_identifier_collision(exc): + suffix = secrets.token_hex(4).upper() # 8 hex chars / 32 bits last_exc = exc continue raise if last_exc is None: raise RuntimeError( - f"project create failed after {PROJECT_CREATE_ATTEMPT_LIMIT} identifier retries " + f"project create failed after {PROJECT_CREATE_ATTEMPT_LIMIT} attempts " f"(prefix={identifier_prefix!r}) with no captured exception" ) raise last_exc @@ -205,17 +239,20 @@ def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[s from .item_types import seed_item_type run_prefix = context["run8"] - name = eval_project_name(run_prefix, second=True) - project = create_project_with_identifier_retry( + variants = eval_project_name_variants(run_prefix, second=True) + name = next(variants) + project = create_project_with_collision_retry( plane, workspace_slug, name=name, identifier_prefix="EB", initial_suffix=run_prefix.upper(), + name_variants=variants, ) context["second_project_id"] = project.id record_seeded_entity(context, "project", project.id) - context["second_project_name"] = name + # The created name, not the requested one: a name collision advances to a variant. + context["second_project_name"] = getattr(project, "name", None) or name context["second_project_identifier"] = getattr(project, "identifier", None) enable_project_features(plane, workspace_slug, project.id) diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index e4f46560..b08b2ccf 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -18,11 +18,13 @@ from evals.core.evidence import TARGET_ENTITY_EVIDENCE from evals.seed import ( R5_TITLE, - create_project_with_identifier_retry, + create_project_with_collision_retry, is_identifier_collision, + is_name_collision, seed_plan, seed_second_project, ) +from evals.seed import projects as projects_mod from evals.tasks.debias import ( L3_TAG_VERSION, L4_PROP_DISPLAY, @@ -1346,7 +1348,7 @@ def create(self, *, workspace_slug, data): suffixes = iter(["AAAAAAAA", "BBBBBBBB"]) monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) - project = create_project_with_identifier_retry( + project = create_project_with_collision_retry( plane, "ws", name="EVAL abcd", @@ -1387,7 +1389,7 @@ def create(self, *, workspace_slug, data): monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( + create_project_with_collision_retry( plane, "ws", name="EVAL x", @@ -1410,7 +1412,7 @@ def create(self, *, workspace_slug, data): plane = MagicMock() plane.projects = Fail500() with pytest.raises(HttpError) as ei: - create_project_with_identifier_retry( + create_project_with_collision_retry( plane, "ws", name="EVAL x", @@ -1426,7 +1428,7 @@ def _identifier_stays_within_plane_limit(_monkeypatch): create=lambda **kwargs: SimpleNamespace(id="project", identifier=kwargs["data"].identifier) ) ) - project = create_project_with_identifier_retry( + project = create_project_with_collision_retry( plane, "ws", name="EVAL x", @@ -1437,7 +1439,7 @@ def _identifier_stays_within_plane_limit(_monkeypatch): assert len(project.identifier) <= seed_mod.PLANE_PROJECT_IDENTIFIER_MAX_LENGTH with pytest.raises(ValueError, match="12-character limit"): - create_project_with_identifier_retry( + create_project_with_collision_retry( plane, "ws", name="EVAL x", @@ -1446,6 +1448,115 @@ def _identifier_stays_within_plane_limit(_monkeypatch): ) +# The observed payload, verbatim: this is what failed L3 in the 177-tool arm on 2026-08-19. +_NAME_TAKEN = "Conflict: name: The project name is already taken" + + +def test_name_and_identifier_collisions_are_told_apart(): + """Both are 409s carrying the same collision wording; only the named field separates them.""" + assert is_name_collision(HttpError(_NAME_TAKEN, 409)) is True + assert is_identifier_collision(HttpError(_NAME_TAKEN, 409)) is False + + assert is_name_collision(HttpError("identifier already taken", 409)) is False + assert is_identifier_collision(HttpError("identifier already taken", 409)) is True + + # A body naming both fields is treated as the identifier case: retrying a suffix is cheap + # and is what this did before names could be retried at all. + both = HttpError("name already taken; identifier already taken", 409) + assert is_name_collision(both) is False + assert is_identifier_collision(both) is True + + # Collision language is still required, and the status still gates it. + assert is_name_collision(HttpError("name is required", 400)) is False + assert is_name_collision(HttpError(_NAME_TAKEN, 500)) is False + + +def _create_project_advances_name_on_name_collision(monkeypatch): + seen: list[tuple[str, str]] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + seen.append((data.name, data.identifier)) + if data.name == "EVAL Delivery Planning Plover": + raise HttpError(_NAME_TAKEN, 409) + # SimpleNamespace, not MagicMock: `name` is reserved on a Mock constructor. + return SimpleNamespace(id="proj-ok", name=data.name, identifier=data.identifier) + + plane = MagicMock() + plane.projects = FakeProjects() + + def _fail_token_hex(_n): + raise AssertionError("a name collision must not regenerate the identifier suffix") + + monkeypatch.setattr(seed_mod.secrets, "token_hex", _fail_token_hex) + + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL Delivery Planning Plover", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + name_variants=iter(["EVAL Delivery Planning Godwit"]), + ) + assert project.name == "EVAL Delivery Planning Godwit" + # Two attempts, same identifier: the suffix was never the problem. + assert seen == [ + ("EVAL Delivery Planning Plover", "EVDEADBEEF"), + ("EVAL Delivery Planning Godwit", "EVDEADBEEF"), + ] + + +def _create_project_name_collision_raises_without_variants(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + attempts.append(data.name) + raise HttpError(_NAME_TAKEN, 409) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: "AAAAAAAA") + + with pytest.raises(HttpError) as ei: + create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="00000000", + ) + assert ei.value.status_code == 409 + # Raised on the first refusal rather than spending the budget on a fine identifier. + assert attempts == ["EVAL x"] + + +def _create_project_exhausts_name_variants(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + attempts.append(data.name) + raise HttpError(_NAME_TAKEN, 409) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: "AAAAAAAA") + + with pytest.raises(HttpError): + create_project_with_collision_retry( + plane, + "ws", + name="EVAL a", + identifier_prefix="EV", + initial_suffix="00000000", + name_variants=iter(["EVAL b", "EVAL c"]), + ) + # Stops when variants run out, well inside the attempt budget. + assert attempts == ["EVAL a", "EVAL b", "EVAL c"] + assert len(attempts) < projects_mod.PROJECT_CREATE_ATTEMPT_LIMIT + + @pytest.mark.parametrize( "case", case_params( @@ -1453,6 +1564,9 @@ def _identifier_stays_within_plane_limit(_monkeypatch): _create_project_raises_after_max_409s, _create_project_non_collision_error_does_not_retry, _identifier_stays_within_plane_limit, + _create_project_advances_name_on_name_collision, + _create_project_name_collision_raises_without_variants, + _create_project_exhausts_name_variants, ), ) def test_create_behaviours(case, monkeypatch): @@ -1467,6 +1581,27 @@ def test_identifier_collision_requires_status_and_language(): assert is_identifier_collision(HttpError("identifier already taken", 500)) is False +def test_project_name_variants_start_deterministic_and_cover_the_pool(): + from evals.core.fixtures import ( + PROJECT_SUFFIX_WORDS, + eval_project_name, + eval_project_name_variants, + ) + + for second in (False, True): + variants = list(eval_project_name_variants("3c128f21", second=second)) + # First name unchanged, so a resumed run and its teardown still agree on it. + assert variants[0] == eval_project_name("3c128f21", second=second) + assert len(variants) == len(PROJECT_SUFFIX_WORDS) + assert len(set(variants)) == len(variants) + + # Reproducible, and the two titles never collide with each other. + assert list(eval_project_name_variants("3c128f21")) == list(eval_project_name_variants("3c128f21")) + assert not set(eval_project_name_variants("3c128f21")) & set(eval_project_name_variants("3c128f21", second=True)) + # Non-hex prefixes fall back to a character sum rather than raising. + assert len(list(eval_project_name_variants("not-hex-at-all"))) == len(PROJECT_SUFFIX_WORDS) + + def _cleanup_dry_run_never_calls_delete(monkeypatch, capsys, _yes): projects = [ SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), @@ -1570,27 +1705,70 @@ def _cleanup_sentinel_mode(monkeypatch, capsys, yes): args = ["--sentinels", "--yes"] if yes else ["--sentinels"] assert cleanup_mod.main(args) == 0 output = capsys.readouterr().out - for fixture_name in ("Acme Corp", "eval-rc1", "Eval Industry", "Incident", "Severity"): + # Bug is a fixture name too, so a leftover Bug type is now a deletion target rather than + # only the type whose Severity property is removed. + for fixture_name in ("Acme Corp", "eval-rc1", "Eval Industry", "Bug", "Incident", "Severity"): assert fixture_name in output assert "Other Corp" not in output assert "Region" not in output + # Reported so a zero match count cannot imply a clean workspace, but not deleted. + assert "Epic" in output + assert "did not create" in output if yes: assert set(delete_calls) == { ("customer", "customer-eval"), ("customer", "customer-short"), ("release_tag", "tag-eval"), ("customer_property", "property-eval"), + ("work_item_type", "type-bug"), ("work_item_type", "type-incident"), ("work_item_property", "severity-eval"), } + assert ("work_item_type", "type-epic") not in set(delete_calls) assert "deleted sentinel" in output assert "would delete sentinel" not in output else: assert delete_calls == [] - assert output.count("would delete sentinel") == 6 + assert output.count("would delete sentinel") == 7 assert "dry-run" in output +def _cleanup_unowned_types_deleted_only_when_asked(monkeypatch, capsys, _yes): + delete_calls: list[tuple[str, str]] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="type-task-a", name="Task"), + SimpleNamespace(id="type-task-b", name="Task"), + ], + properties=SimpleNamespace(list=lambda **kw: []), + delete=lambda **kw: delete_calls.append(("work_item_type", kw["type_id"])), + ), + workspace_work_item_properties=SimpleNamespace(list=lambda **kw: []), + ) + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + # No fixture-named types: without --unowned this reports nothing to delete, but must still + # surface the two it will not touch. This is the exact shape that read as clean before. + assert cleanup_mod.main(["--sentinels", "--yes"]) == 0 + output = capsys.readouterr().out + assert "sentinel_matches=0" in output + assert "nothing to delete" in output + assert output.count("'Task'") == 2 + assert delete_calls == [] + + assert cleanup_mod.main(["--sentinels", "--unowned", "--yes"]) == 0 + assert set(delete_calls) == {("work_item_type", "type-task-a"), ("work_item_type", "type-task-b")} + + # --unowned is meaningless for the project cleaner and must not be silently ignored. + assert cleanup_mod.main(["--unowned"]) == 2 + + @pytest.mark.parametrize( ("case", "yes"), [ @@ -1598,6 +1776,7 @@ def _cleanup_sentinel_mode(monkeypatch, capsys, yes): pytest.param(_cleanup_yes_deletes, None, id="project-delete"), pytest.param(_cleanup_sentinel_mode, False, id="sentinel-dry-run"), pytest.param(_cleanup_sentinel_mode, True, id="sentinel-delete"), + pytest.param(_cleanup_unowned_types_deleted_only_when_asked, None, id="sentinel-unowned"), ], ) def test_cleanup_behaviours(case, yes, monkeypatch, capsys): From d5cb99506f077c2a7c428f158e24145004d79dfa Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Thu, 20 Aug 2026 22:50:43 +0530 Subject: [PATCH 76/93] Record the request beside a recorded result W7 fails reproducibly at rep 1: workitem_link.create returns success, and the target work item then has no links at all. Two explanations fit -- the agent linked the wrong item, or create reports success without persisting -- and they could not be told apart, because rows keep args_chars and not the request. A recorded refusal that cannot be attributed to a target answers half a question, and the second explanation would be a defect in the server this harness exists to measure. Args now ride along with a recorded payload and stay out otherwise. Keyed off result_text rather than a new flag: that is already the signal payload recording is on, and it keeps the two halves of one record together by construction. Default behaviour is unchanged -- args remain metrics-only, with action still kept unconditionally since it is half the tool choice on an action-dispatch surface. The flag's help now says it records requests too, so the wider capture is not a hidden effect of a name that mentions only results. Tested across the same hop chain as error_class -- proxy row, sidecar reader, agent_run_to_task_result, to_row, from_row -- because that chain is where a field of this kind gets silently dropped: the sidecar reader rebuilds calls from an explicit key list, and to_row writes an explicit dict. Confirmed the test fails when the to_row hop is removed rather than only passing against the finished code. Co-Authored-By: Claude Opus 5 (1M context) --- evals/cli.py | 5 ++-- evals/core/results.py | 16 ++++++++++ tests/evals/test_error_class.py | 52 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/evals/cli.py b/evals/cli.py index 3c17f3be..b3afc71f 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -143,8 +143,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--record-result-payloads", action="store_true", help=( - "CLI drivers only: record serialized tool-result text for tokenizer counting " - "(off by default; sidecars may contain live workspace data)" + "CLI drivers only: record serialized tool-result text for tokenizer counting, and " + "the request args beside it so a recorded result can be attributed to its target " + "(off by default; sidecars and rows may contain live workspace data)" ), ) p.add_argument("--out", type=str, default=None, help="JSONL output path") diff --git a/evals/core/results.py b/evals/core/results.py index 6e830e15..bc93204a 100644 --- a/evals/core/results.py +++ b/evals/core/results.py @@ -109,6 +109,10 @@ class CallRecord: raw_tool: str | None = None # Which kind of "no" an errored call received; None when the call succeeded. error_class: str | None = None + # The request body, recorded only under --record-result-payloads. Args are metrics-only + # by default (see args_chars); without the request, a recorded refusal can be read but + # not attributed to the target it names, which is the question a payload is kept to answer. + args_json: str | None = None result_tokens_skipped: str | None = None # None means the response was not checked; [] means checked with no match. observed_sentinels: list[str] | None = None @@ -276,6 +280,8 @@ def usage_row(item: Usage) -> dict[str, int]: item["result_tokens_skipped"] = call.result_tokens_skipped if call.error_class is not None: item["error_class"] = call.error_class + if call.args_json is not None: + item["args_json"] = call.args_json if call.observed_sentinels is not None: item["observed_sentinels"] = list(call.observed_sentinels) calls.append(item) @@ -379,6 +385,7 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ), duration_ms=raw.get("duration_ms"), action=(str(raw["action"]) if raw.get("action") is not None else None), + args_json=(str(raw["args_json"]) if raw.get("args_json") is not None else None), result_tokens_skipped=( str(raw["result_tokens_skipped"]) if raw.get("result_tokens_skipped") is not None else None ), @@ -588,6 +595,15 @@ def agent_run_to_task_result( # tool choice — keep it (args content is otherwise not persisted). if isinstance(args, dict) and isinstance(args.get("action"), str): rec.action = args["action"] + # Under --record-result-payloads the proxy has already put the result body on the row, + # so the request that produced it is the other half of the same record. Keyed off + # result_text rather than a new flag: that is the signal payload recording is on, and + # a recorded response with no recorded request cannot be attributed to a target. + if isinstance(c.get("result_text"), str) and isinstance(args, dict) and args: + try: + rec.args_json = json.dumps(args, default=str, ensure_ascii=False) + except Exception: + rec.args_json = str(args) calls.append(rec) client_tool_calls: list[CallRecord] = [] diff --git a/tests/evals/test_error_class.py b/tests/evals/test_error_class.py index ce56c071..f6c9e627 100644 --- a/tests/evals/test_error_class.py +++ b/tests/evals/test_error_class.py @@ -224,6 +224,58 @@ def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): assert split_errors(reloaded.calls)["surface"] == 1, "the report did not see it" +def test_request_args_are_recorded_only_alongside_a_recorded_result(tmp_path): + """A recorded refusal that cannot be attributed to a target answers half a question. + + W7 failed reproducibly with workitem_link.create reporting success while the link was + absent, and the two candidate explanations -- wrong target, or a create that does not + persist -- were indistinguishable because only args_chars was kept. Args now ride along + with a recorded payload, and stay out when payloads are off: the same hop chain as + error_class, which is where a field of this kind gets silently dropped. + """ + import json + + from evals.core.results import AgentRun, TaskResult, Usage, agent_run_to_task_result + from evals.drivers.cli.sidecar import load_proxy_sidecar + + args = {"action": "create", "workitem_id": "wi-42", "url": "https://example.com/eval/runbook-w7"} + + def row_for(*, with_payload: bool) -> dict: + row = { + "tool": "workitem_link", + "args": args, + "is_error": False, + "result_chars": 88, + "duration_ms": 12, + "seq": 1, + } + if with_payload: + row["result_text"] = '{"content":[{"type":"text","text":"link created"}]}' + return row + + def roundtrip(*, with_payload: bool) -> TaskResult: + sidecar = tmp_path / f"sidecar-{with_payload}.jsonl" + sidecar.write_text(json.dumps(row_for(with_payload=with_payload)) + "\n", encoding="utf-8") + calls, _status = load_proxy_sidecar(sidecar) + assert calls, "the sidecar reader produced no calls" + result = agent_run_to_task_result( + AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn") + ) + return TaskResult.from_row(json.loads(json.dumps(result.to_row()))) + + recorded = roundtrip(with_payload=True) + assert recorded.calls[0].args_json is not None, "args were dropped somewhere in the chain" + assert json.loads(recorded.calls[0].args_json) == args + # The target is the point: without it the record cannot say which item was linked. + assert "wi-42" in recorded.calls[0].args_json + + # Default stays metrics-only. args_chars is still there; the body is not. + plain = roundtrip(with_payload=False) + assert plain.calls[0].args_json is None, "args leaked into a run that did not ask for payloads" + assert plain.calls[0].args_chars > 0 + assert plain.calls[0].action == "create", "action is kept regardless — it is half the tool choice" + + def test_a_refusal_the_server_called_a_success_is_still_counted(): """The server answers a malformed call with a plain result whose text says "Error", so the protocol reports success. Roughly 47 per battery arrived that From 8978f2a69d41e1f4b3335961171d70b919b538c7 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 21 Aug 2026 16:04:13 +0530 Subject: [PATCH 77/93] Adopt the project an ambiguous create leaves behind The leftover projects were not a teardown bug. Teardown deletes by the id in its context, and on a create that raised, the id was never returned, so the context never held it and teardown was never asked to delete anything -- reporting cleanup_error 0 truthfully while a project sat in the workspace skewing every later workspace-wide task. The creates in question raised a client-side read timeout at 30s against the API under load, after the server had already made the project. Evidence: the three surviving orphans date from the contended run (two of them 16 seconds apart in different workspaces, which is one API stall, not two teardown failures), and the later run on a quiet machine created none. An exception with no HTTP status means the outcome was never learned, so the identifier -- which is unique per workspace -- is now looked up before giving up. If the project is there the create did succeed and it is adopted, which both hands teardown the id and turns a lost row into a normal one. Deliberately not extended to 5xx: those are ambiguous in principle, but the measured failure is a read timeout, and adopting after any HTTP error would adopt following refusals that created nothing. Tested for all three outcomes an ambiguous create has -- present and adopted, absent and re-raised, and an HTTP refusal that must not even look. Confirmed the adopt test fails with the guard disabled rather than only passing against the finished code. Co-Authored-By: Claude Opus 5 (1M context) --- evals/seed/projects.py | 41 +++++++++++++++++ tests/evals/seed/test_seed.py | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/evals/seed/projects.py b/evals/seed/projects.py index e9fc927f..e51d61a8 100644 --- a/evals/seed/projects.py +++ b/evals/seed/projects.py @@ -9,6 +9,7 @@ from plane import PlaneClient from plane.errors.errors import HttpError from plane.models.projects import CreateProject, ProjectFeature, UpdateProject +from plane.models.query_params import PaginatedQueryParams from plane.models.work_items import CreateWorkItem from plane.models.workspaces import WorkspaceFeature @@ -70,6 +71,28 @@ def is_identifier_collision(exc: BaseException) -> bool: return _is_collision(exc) and not is_name_collision(exc) +def find_project_by_identifier(plane: PlaneClient, workspace_slug: str, identifier: str) -> Any | None: + """Return the project holding this exact identifier, or None. + + Identifiers are unique per workspace, so this settles the one question an ambiguous + create leaves open: did the server create it before the client stopped waiting? + """ + cursor = None + while True: + page = plane.projects.list( + workspace_slug=workspace_slug, + params=PaginatedQueryParams(per_page=100, cursor=cursor), + ) + results = page.results if hasattr(page, "results") else page + for proj in results or []: + if (getattr(proj, "identifier", None) or "").strip().upper() == identifier.strip().upper(): + return proj + # The SDK always populates next_cursor, so paging must stop on next_page_results. + if not getattr(page, "next_page_results", False): + return None + cursor = page.next_cursor + + def create_project_with_collision_retry( plane: PlaneClient, workspace_slug: str, @@ -120,6 +143,24 @@ def create_project_with_collision_retry( suffix = secrets.token_hex(4).upper() # 8 hex chars / 32 bits last_exc = exc continue + # An exception carrying no HTTP status means the client never learned the outcome: + # the server may well have created the project before the read timed out. That is + # how the orphans got there. The id was never returned, so the caller never put it + # in the teardown context, so teardown was never asked to delete it -- and reported + # cleanup_error 0 truthfully while a project sat in the workspace, skewing every + # later workspace-wide task. Adopting the project both removes the orphan and turns + # a lost row into a normal one. + # + # Limited to no-response errors on purpose. A 5xx is also ambiguous in principle, + # but the measured failure is a client-side read timeout, and treating every HTTP + # error as maybe-created would adopt projects after refusals that created nothing. + if not isinstance(exc, HttpError): + try: + adopted = find_project_by_identifier(plane, workspace_slug, identifier) + except Exception: + adopted = None # Lookup failed too; report the original failure. + if adopted is not None: + return adopted raise if last_exc is None: raise RuntimeError( diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py index b08b2ccf..8286d93d 100644 --- a/tests/evals/seed/test_seed.py +++ b/tests/evals/seed/test_seed.py @@ -1531,6 +1531,88 @@ def create(self, *, workspace_slug, data): assert attempts == ["EVAL x"] +class _ReadTimeout(Exception): + """Stands in for requests' ReadTimeout: an error carrying no HTTP status.""" + + +def _create_project_adopts_the_project_a_timeout_orphaned(monkeypatch): + """A create that times out after the server made the project must not orphan it. + + This is where the leftover projects came from: the client stopped waiting, the id was + never returned, so the caller never put it in the teardown context and teardown reported + cleanup_error 0 while the project sat in the workspace. + """ + listed: list[str] = [] + + class TimeoutThenPresent: + def create(self, *, workspace_slug, data): + raise _ReadTimeout("HTTPConnectionPool(host='localhost', port=8000): Read timed out.") + + def list(self, *, workspace_slug, params=None): + listed.append(workspace_slug) + return SimpleNamespace( + results=[ + SimpleNamespace(id="other", identifier="EVZZZZZZZZ", name="EVAL other"), + SimpleNamespace(id="orphan", identifier="EVDEADBEEF", name="EVAL Delivery Planning Wren"), + ], + next_page_results=False, + next_cursor="100:0:0", + ) + + plane = MagicMock() + plane.projects = TimeoutThenPresent() + + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL Delivery Planning Wren", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + ) + # Adopted by identifier, so the caller learns the id and teardown can delete it. + assert project.id == "orphan" + assert listed == ["ws"] + + +def _create_project_reraises_when_nothing_was_created(monkeypatch): + """A timeout where the server created nothing must still fail, not invent a project.""" + + class TimeoutAndAbsent: + def create(self, *, workspace_slug, data): + raise _ReadTimeout("Read timed out.") + + def list(self, *, workspace_slug, params=None): + return SimpleNamespace(results=[], next_page_results=False, next_cursor="100:0:0") + + plane = MagicMock() + plane.projects = TimeoutAndAbsent() + with pytest.raises(_ReadTimeout): + create_project_with_collision_retry( + plane, "ws", name="EVAL x", identifier_prefix="EV", initial_suffix="00000000" + ) + + +def _create_project_does_not_adopt_after_an_http_refusal(monkeypatch): + """An HTTP error is a known outcome: nothing was created, so do not go looking.""" + listed: list[str] = [] + + class Refuses: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + def list(self, *, workspace_slug, params=None): + listed.append(workspace_slug) + return SimpleNamespace(results=[], next_page_results=False, next_cursor=None) + + plane = MagicMock() + plane.projects = Refuses() + with pytest.raises(HttpError): + create_project_with_collision_retry( + plane, "ws", name="EVAL x", identifier_prefix="EV", initial_suffix="00000000" + ) + assert listed == [], "an HTTP refusal must not trigger an adoption lookup" + + def _create_project_exhausts_name_variants(monkeypatch): attempts: list[str] = [] @@ -1567,6 +1649,9 @@ def create(self, *, workspace_slug, data): _create_project_advances_name_on_name_collision, _create_project_name_collision_raises_without_variants, _create_project_exhausts_name_variants, + _create_project_adopts_the_project_a_timeout_orphaned, + _create_project_reraises_when_nothing_was_created, + _create_project_does_not_adopt_after_an_http_refusal, ), ) def test_create_behaviours(case, monkeypatch): From 824ef67c7e71ff1eb8a8ddbaf50352bfe1f13576 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 21 Aug 2026 16:05:11 +0530 Subject: [PATCH 78/93] Document the success-flagged refusal count and recorded request args Both shipped without their documentation. The refusal-counted-as-success line is the one a reader is most likely to misread as a duplicate of the errored-call total, so it says why it cannot join that total and what it was worth catching: a surface measuring 12.8% refusals was really near 28%. The args note says what recording buys, since args are otherwise deliberately metrics-only. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 14 ++++++++++++++ evals/README.md | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 1716318b..6fdb4fcb 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -100,6 +100,20 @@ additive. Status outranks wording: a 404 whose body mentions a missing argument absent resource, since only a payload with no status at all can be a call that never reached the API. The split cannot charge a surface for misleading an agent into a single wrong lookup. +A refusal the server reports as a **successful** result is classified too, and counted in the +split while staying out of the errored-call total, which is keyed on the protocol's error flag. +Reports state that count separately (`N refusal(s) arrived flagged as successful results`) so the +two never silently merge. Without it the metric was blind to roughly a third of what agents +actually get told no about: a run measuring 12.8% refusals was really near 28%. Detection is +deliberately narrow — only wording this server owns, and the stray-argument form must carry both +halves of its sentence — so a result that merely quotes a refusal is not counted as one. + +Recorded payloads carry the **request** as well (`args_json`, under +`--record-result-payloads`). Args are otherwise metrics-only: a recorded refusal that cannot be +attributed to the target it names answers half a question. This is what separated an agent +linking the wrong work item from a create that does not persist, when both fit the same +symptom. + Single-run success headlines use the same sampling unit: each evaluated task contributes its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. The pooled repetition rate and its Wilson interval remain visible as a descriptive figure, diff --git a/evals/README.md b/evals/README.md index dc7444e7..90524276 100644 --- a/evals/README.md +++ b/evals/README.md @@ -180,6 +180,15 @@ first absent read, which is an answer rather than an obstacle), plus `other` and friction never stands in for nothing having been classified — including when reading a result file recorded before the split existed. See DESIGN.md for the classification rules. +A refusal the server hands back as a *successful* result is counted too, reported on its own +line (`N refusal(s) arrived flagged as successful results`) because it cannot join a total keyed +on the protocol's error flag. It is worth watching: one measured surface refused roughly twice +as often as its errored-call count implied. + +`--record-result-payloads` keeps the request args beside each recorded result. Args are +metrics-only by default (`args_chars`), and a recorded result whose target is unknown cannot say +*which* item a call acted on — which is exactly the question a failing write raises. + Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and fixture names. The battery contains exactly what the agent is asked and no expectation about From 8e1681eb496df33c2ed54680ae4294704e406c06 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Fri, 21 Aug 2026 20:59:12 +0530 Subject: [PATCH 79/93] Declare the OpenAI eval provider's package `openai` is in KNOWN_API_PROVIDERS and has a driver, but its package was never declared, so --provider openai was a provider the harness advertised and could not run. The driver already fails with a clear message rather than an ImportError, which is why this went unnoticed: nothing breaks until someone selects the provider. Kept as its own extra rather than folded into `evals`, so an anthropic-only install is not made to carry a second vendor's client. Install with `.[evals,evals-openai]`. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8643becb..5fe9068f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,13 @@ dev = [ evals = [ "anthropic>=0.121.0", ] +# The OpenAI API provider is a second vendor's client, so it stays opt-in rather than being +# imposed on an anthropic-only install. Declared all the same: without it, `--provider openai` +# is a provider the harness advertises in KNOWN_API_PROVIDERS and cannot run, and a result +# anyone else has to reproduce should not depend on remembering an undeclared package. +evals-openai = [ + "openai>=1.0.0", +] [project.scripts] plane-mcp-server = "plane_mcp.__main__:main" From 16b0e80ff2e31d64fa0a5ab2db32cf3bde6b9b1b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 24 Aug 2026 20:31:10 +0530 Subject: [PATCH 80/93] Move the OpenAI eval backend to /v1/responses Chat Completions cannot run this harness on current models. gpt-5.6-luna answers any request carrying function tools with `400 -- Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'`. The harness always sends tools, so every turn of every row failed; the provider had evidently never been exercised against these models, which fits its package not having been declared either. Setting reasoning_effort='none' would also clear the error and was rejected: it silences reasoning here while the vendor CLI drivers keep it, so an API arm could no longer be compared against a CLI arm of the same model -- the one question an API arm exists to answer. Producing a clean-looking table that answers a different question is worse than not running. The wire differences, all of them load-bearing: tools are declared flat rather than nested under "function"; the system prompt is `instructions`, not a message; the cap is `max_output_tokens`; there is no finish_reason, so the stop reason comes from status plus what the turn emitted, with tool calls outranking status because a completed response carrying calls is the loop's continue signal; results go back as `function_call_output` keyed by `call_id`, not as role-tagged tool messages. The model's own output items are replayed verbatim, reasoning items included -- these models expect that chain intact on the next request. Verified live end to end against the real 28-tool surface: tool_use, a tool round-trip, then end_turn with usage and the realized model captured. Offline tests rewritten to the Responses shape, with the request assertions confirmed to fail against the old nested tool shape and against dropping reasoning items, rather than only passing against the new code. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/api/openai.py | 241 ++++++++++++++++--------- tests/evals/drivers/test_api_driver.py | 184 +++++++++---------- 2 files changed, 250 insertions(+), 175 deletions(-) diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py index f229afdf..c60a37c2 100644 --- a/evals/drivers/api/openai.py +++ b/evals/drivers/api/openai.py @@ -1,4 +1,16 @@ -"""OpenAI Chat Completions translation for the provider-neutral eval loop.""" +"""OpenAI Responses translation for the provider-neutral eval loop. + +Responses rather than Chat Completions because Chat Completions cannot run this harness at +all on current models: ``gpt-5.6-luna`` answers a request carrying function tools with +``400 — Function tools with reasoning_effort are not supported for gpt-5.6-luna in +/v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to +'none'``. The harness always sends tools, so every turn failed. + +Setting ``reasoning_effort='none'`` would also have satisfied that error, and was rejected: +it would silence reasoning on this path while the vendor CLI drivers keep it, so an arm run +here could not be compared against a CLI arm of the same model — which is the one question +an API arm exists to answer. +""" from __future__ import annotations @@ -23,32 +35,59 @@ def _field(value: Any, name: str, default: Any = None) -> Any: def _normalize_usage(usage: Any) -> Usage | None: + """Map Responses usage onto the neutral shape. + + Responses names these ``input_tokens``/``output_tokens``, where Chat Completions said + ``prompt_tokens``/``completion_tokens``; both spellings are read so an injected fake or a + future field rename does not silently report zero. + """ if usage is None: return None - prompt_details = _field(usage, "prompt_tokens_details") + input_details = _field(usage, "input_tokens_details") or _field(usage, "prompt_tokens_details") + input_tokens = _field(usage, "input_tokens") + if input_tokens is None: + input_tokens = _field(usage, "prompt_tokens", 0) + output_tokens = _field(usage, "output_tokens") + if output_tokens is None: + output_tokens = _field(usage, "completion_tokens", 0) return Usage( - input_tokens=int(_field(usage, "prompt_tokens", 0) or 0), - output_tokens=int(_field(usage, "completion_tokens", 0) or 0), - cache_read_input_tokens=int(_field(prompt_details, "cached_tokens", 0) or 0), + input_tokens=int(input_tokens or 0), + output_tokens=int(output_tokens or 0), + cache_read_input_tokens=int(_field(input_details, "cached_tokens", 0) or 0), ) -def _normalize_stop_reason(finish_reason: Any, refusal: Any) -> tuple[StopReason, str | None]: - raw = str(finish_reason) if finish_reason is not None else None - if refusal or finish_reason == "content_filter": - return StopReason.REFUSAL, raw - reason = { - "stop": StopReason.END_TURN, - "length": StopReason.MAX_TOKENS, - "tool_calls": StopReason.TOOL_USE, - # Retain support for the predecessor of ``tool_calls``. - "function_call": StopReason.TOOL_USE, - }.get(raw, StopReason.UNKNOWN) - return reason, raw +def _text_from_content(content: Any) -> str: + """Concatenate the text parts of one output message's content list.""" + if isinstance(content, str): + return content + parts: list[str] = [] + for part in content or (): + kind = str(_field(part, "type", "") or "") + if kind in ("output_text", "text"): + parts.append(str(_field(part, "text", "") or "")) + elif kind == "refusal": + parts.append(str(_field(part, "refusal", "") or "")) + return "".join(parts) + + +def _parse_arguments(raw_args: Any) -> tuple[dict[str, Any], str]: + """Return (args dict, wire string). Malformed JSON is preserved, never dropped.""" + if isinstance(raw_args, str): + try: + parsed = json.loads(raw_args or "{}") + except json.JSONDecodeError: + return {"_raw": raw_args}, raw_args + if not isinstance(parsed, dict): + return {"_raw": parsed}, raw_args + return parsed, raw_args + if isinstance(raw_args, dict): + return raw_args, json.dumps(raw_args, separators=(",", ":")) + return {"_raw": raw_args}, json.dumps(raw_args, default=str) class OpenAIBackend: - """Stateful adapter over ``client.chat.completions.create``. + """Stateful adapter over ``client.responses.create``. ``openai`` is deliberately imported only when no client was injected, so importing this module and all offline tests work without that package. @@ -63,7 +102,7 @@ def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> except ImportError as exc: raise RuntimeError( "the OpenAI API provider requires the optional 'openai' package; " - "install it in the runtime that launches evals" + "install it with the 'evals-openai' extra" ) from exc client = OpenAI() @@ -71,23 +110,24 @@ def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> self.model = model self.actual_model = model self.max_tokens = max_tokens - self.messages: list[dict[str, Any]] = [] + # Responses calls this the input list; it carries user/assistant items plus + # function_call and function_call_output items, not role-tagged tool messages. + self.input_items: list[Any] = [] + self.instructions: str | None = None self.tools: list[dict[str, Any]] = [] self.started = False def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: - self.messages = [] - if system is not None: - self.messages.append({"role": "system", "content": system}) - self.messages.append({"role": "user", "content": prompt}) + # The system prompt is a top-level field here rather than a message in the list. + self.instructions = system + self.input_items = [{"role": "user", "content": prompt}] + # Responses declares a function tool flat; Chat Completions nested it under "function". self.tools = [ { "type": "function", - "function": { - "name": tool.name, - "description": tool.description, - "parameters": tool.input_schema, - }, + "name": tool.name, + "description": tool.description, + "parameters": tool.input_schema, } for tool in tools ] @@ -96,73 +136,106 @@ def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: def next_turn(self) -> Turn: if not self.started: raise RuntimeError("OpenAIBackend.start() must be called before next_turn()") - completion = self.client.chat.completions.create( - model=self.model, - max_completion_tokens=self.max_tokens, - messages=self.messages, - tools=self.tools, - ) - choices = _field(completion, "choices", None) or [] - if not choices: - raise RuntimeError("OpenAI Chat Completions returned no choices") - choice = choices[0] - message = _field(choice, "message") - raw_calls = _field(message, "tool_calls", None) or [] - + request: dict[str, Any] = { + "model": self.model, + "max_output_tokens": self.max_tokens, + "input": self.input_items, + } + if self.instructions is not None: + request["instructions"] = self.instructions + if self.tools: + request["tools"] = self.tools + response = self.client.responses.create(**request) + + output = _field(response, "output", None) or [] calls: list[ToolCall] = [] - wire_calls: list[dict[str, Any]] = [] - for raw_call in raw_calls: - function = _field(raw_call, "function") - raw_args = _field(function, "arguments", "{}") or "{}" - if isinstance(raw_args, str): - try: - args = json.loads(raw_args) - except json.JSONDecodeError: - args = {"_raw": raw_args} - elif isinstance(raw_args, dict): - args = raw_args - raw_args = json.dumps(raw_args, separators=(",", ":")) + texts: list[str] = [] + refusal_text = "" + for item in output: + kind = str(_field(item, "type", "") or "") + if kind == "function_call": + args, wire_args = _parse_arguments(_field(item, "arguments", "{}")) + # call_id is the identifier a function_call_output must echo back; id is the + # item's own handle. Only call_id closes the loop. + call_id = str(_field(item, "call_id", "") or _field(item, "id", "") or "") + calls.append(ToolCall(id=call_id, name=str(_field(item, "name", "") or ""), args=args)) + # Echo the model's own item back verbatim so the provider sees the history it + # produced, rather than a reconstruction of it. + self.input_items.append(item) + elif kind in ("message", ""): + content = _field(item, "content") + texts.append(_text_from_content(content)) + self.input_items.append(item) else: - args = {"_raw": raw_args} - raw_args = json.dumps(raw_args, default=str) - if not isinstance(args, dict): - args = {"_raw": args} - call_id = str(_field(raw_call, "id", "") or "") - name = str(_field(function, "name", "") or "") - calls.append(ToolCall(id=call_id, name=name, args=args)) - wire_calls.append( - { - "id": call_id, - "type": "function", - "function": {"name": name, "arguments": raw_args}, - } - ) - - content = _field(message, "content") - refusal = _field(message, "refusal") - assistant_message: dict[str, Any] = {"role": "assistant", "content": content} - if wire_calls: - assistant_message["tool_calls"] = wire_calls - self.messages.append(assistant_message) - - response_model = _field(completion, "model") + # Reasoning and any future item type: replayed untouched. Dropping a reasoning + # item breaks the chain these models expect on the next request. + self.input_items.append(item) + + for item in output: + if str(_field(item, "type", "") or "") == "message": + for part in _field(item, "content") or (): + if str(_field(part, "type", "") or "") == "refusal": + refusal_text = str(_field(part, "refusal", "") or "") + + response_model = _field(response, "model") if response_model: self.actual_model = str(response_model) - stop_reason, provider_stop_reason = _normalize_stop_reason(_field(choice, "finish_reason"), refusal) + + status = str(_field(response, "status", "") or "") + incomplete = _field(response, "incomplete_details") + incomplete_reason = str(_field(incomplete, "reason", "") or "") if incomplete else "" + stop_reason, provider_stop_reason = self._stop_reason( + status=status, + incomplete_reason=incomplete_reason, + has_calls=bool(calls), + refusal=refusal_text, + ) + text = "".join(texts) or refusal_text + if not text: + # output_text is the SDK's own concatenation; only consulted as a fallback so a + # shape this adapter does not model yet still yields the answer. + text = str(_field(response, "output_text", "") or "") return Turn( - text=str(content or refusal or ""), + text=text, tool_calls=calls, - usage=_normalize_usage(_field(completion, "usage")), + usage=_normalize_usage(_field(response, "usage")), stop_reason=stop_reason, provider_stop_reason=provider_stop_reason, ) + @staticmethod + def _stop_reason( + *, + status: str, + incomplete_reason: str, + has_calls: bool, + refusal: str, + ) -> tuple[StopReason, str | None]: + """Derive the neutral stop reason. + + Responses has no finish_reason: a turn's outcome is its status plus what it emitted. + Tool calls win over status because a completed response carrying calls is the loop's + continue signal, which is what TOOL_USE means to the driver. + """ + raw = incomplete_reason or status or None + if refusal: + return StopReason.REFUSAL, raw + if has_calls: + return StopReason.TOOL_USE, raw + if incomplete_reason == "max_output_tokens": + return StopReason.MAX_TOKENS, raw + if incomplete_reason == "content_filter": + return StopReason.REFUSAL, raw + if status == "completed": + return StopReason.END_TURN, raw + return StopReason.UNKNOWN, raw + def add_tool_results(self, results: list[ToolResult]) -> None: - self.messages.extend( + self.input_items.extend( { - "role": "tool", - "tool_call_id": result.call_id, - "content": result.text, + "type": "function_call_output", + "call_id": result.call_id, + "output": result.text, } for result in results ) diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 7e236726..57c9f5a5 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -135,7 +135,10 @@ def create(self, **kwargs): return self.responses.popleft() -class FakeOpenAICompletions: +class FakeOpenAIResponses: + """Stands in for ``client.responses``. Deep-copies each request, because the backend + appends the model's own output items to the same input list it sends.""" + def __init__(self, responses: list[dict[str, Any]]) -> None: self.responses = deque(responses) self.requests: list[dict[str, Any]] = [] @@ -145,6 +148,10 @@ def create(self, **kwargs): return self.responses.popleft() +def openai_client(responses: FakeOpenAIResponses) -> SimpleNamespace: + return SimpleNamespace(responses=responses) + + def test_registered_third_party_backend_runs_without_driver_changes(): created: list[FakeBackend] = [] @@ -666,77 +673,67 @@ def test_anthropic_backend_translates_tools_turns_and_results(): @pytest.mark.parametrize( - ("raw_reason", "expected"), + ("status", "incomplete_reason", "expected"), [ - ("stop", StopReason.END_TURN), - ("tool_calls", StopReason.TOOL_USE), - ("length", StopReason.MAX_TOKENS), - ("content_filter", StopReason.REFUSAL), - ("future_reason", StopReason.UNKNOWN), + ("completed", None, StopReason.END_TURN), + ("incomplete", "max_output_tokens", StopReason.MAX_TOKENS), + ("incomplete", "content_filter", StopReason.REFUSAL), + ("failed", None, StopReason.UNKNOWN), + ("queued", None, StopReason.UNKNOWN), ], ) -def test_openai_backend_normalizes_and_preserves_stop_reason(raw_reason, expected): - completions = FakeOpenAICompletions( +def test_openai_backend_derives_stop_reason_from_status(status, incomplete_reason, expected): + """Responses has no finish_reason: the outcome is status plus what the turn emitted.""" + responses = FakeOpenAIResponses( [ { "model": "gpt", - "choices": [ - { - "finish_reason": raw_reason, - "message": {"content": "done", "tool_calls": []}, - } - ], + "status": status, + "incomplete_details": ({"reason": incomplete_reason} if incomplete_reason else None), + "output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}], "usage": None, } ] ) - backend = OpenAIBackend( - "gpt", - max_tokens=10, - client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), - ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(responses)) backend.start(None, "prompt", []) turn = backend.next_turn() assert turn.stop_reason is expected - assert turn.provider_stop_reason == raw_reason + assert turn.provider_stop_reason == (incomplete_reason or status) -def _openai_backend_translates_tools_calls_and_tool_messages(): +def _openai_backend_translates_tools_calls_and_outputs(): responses = [ { "model": "gpt-actual", - "choices": [ + "status": "completed", + "output": [ + {"type": "reasoning", "id": "rs-1", "summary": []}, { - "finish_reason": "tool_calls", - "message": { - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, - } - ], - }, - } + "type": "function_call", + "id": "fc-1", + "call_id": "call-1", + "name": "lookup", + "arguments": '{"q":"x"}', + }, ], "usage": { - "prompt_tokens": 12, - "completion_tokens": 3, - "prompt_tokens_details": {"cached_tokens": 5}, + "input_tokens": 12, + "output_tokens": 3, + "input_tokens_details": {"cached_tokens": 5}, }, }, { "model": "gpt-actual", - "choices": [{"finish_reason": "stop", "message": {"content": "done", "tool_calls": []}}], - "usage": {"prompt_tokens": 20, "completion_tokens": 4}, + "status": "completed", + "output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}], + "usage": {"input_tokens": 20, "output_tokens": 4}, }, ] - completions = FakeOpenAICompletions(responses) - client = SimpleNamespace(chat=SimpleNamespace(completions=completions)) - backend = OpenAIBackend("gpt-requested", max_tokens=321, client=client) + fake = FakeOpenAIResponses(responses) + backend = OpenAIBackend("gpt-requested", max_tokens=321, client=openai_client(fake)) tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) backend.start("system", "prompt", [tool]) @@ -744,90 +741,95 @@ def _openai_backend_translates_tools_calls_and_tool_messages(): backend.add_tool_results([ToolResult("call-1", "value")]) second = backend.next_turn() - first_request = completions.requests[0] - assert first_request["max_completion_tokens"] == 321 - assert first_request["messages"] == [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "prompt"}, - ] + first_request = fake.requests[0] + # Responses names the cap max_output_tokens and carries the system prompt as instructions. + assert first_request["max_output_tokens"] == 321 + assert first_request["instructions"] == "system" + assert first_request["input"] == [{"role": "user", "content": "prompt"}] + # A function tool is declared flat here; Chat Completions nested it under "function". assert first_request["tools"] == [ { "type": "function", - "function": { - "name": "lookup", - "description": "Look up", - "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, - }, + "name": "lookup", + "description": "Look up", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, } ] assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] assert first.stop_reason is StopReason.TOOL_USE - assert first.provider_stop_reason == "tool_calls" assert first.usage == Usage(12, 3, 5, 0) - second_messages = completions.requests[1]["messages"] - assert second_messages[2] == { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": {"name": "lookup", "arguments": '{"q":"x"}'}, - } - ], - } - assert second_messages[3] == {"role": "tool", "tool_call_id": "call-1", "content": "value"} + + second_input = fake.requests[1]["input"] + # The model's own items are replayed verbatim, reasoning included: dropping a reasoning + # item breaks the chain these models expect on the next request. + assert second_input[1]["type"] == "reasoning" + assert second_input[2]["type"] == "function_call" + assert second_input[2]["call_id"] == "call-1" + # The result is a function_call_output keyed by call_id, not a role-tagged tool message. + assert second_input[3] == {"type": "function_call_output", "call_id": "call-1", "output": "value"} assert second.text == "done" assert second.stop_reason is StopReason.END_TURN - assert second.provider_stop_reason == "stop" assert backend.actual_model == "gpt-actual" def _openai_backend_normalizes_refusal_for_driver_guard(): - completions = FakeOpenAICompletions( + """A refusal must outrank the tool calls beside it, or the driver executes a refused write.""" + fake = FakeOpenAIResponses( [ { "model": "gpt", - "choices": [ + "status": "completed", + "output": [ + {"type": "message", "content": [{"type": "refusal", "refusal": "declined"}]}, { - "finish_reason": "content_filter", - "message": { - "content": None, - "refusal": "declined", - "tool_calls": [ - { - "id": "danger", - "type": "function", - "function": {"name": "write", "arguments": "{}"}, - } - ], - }, - } + "type": "function_call", + "call_id": "danger", + "name": "write", + "arguments": "{}", + }, ], "usage": None, } ] ) - backend = OpenAIBackend( - "gpt", - max_tokens=10, - client=SimpleNamespace(chat=SimpleNamespace(completions=completions)), - ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(fake)) backend.start(None, "prompt", []) turn = backend.next_turn() assert turn.stop_reason is StopReason.REFUSAL - assert turn.provider_stop_reason == "content_filter" assert turn.text == "declined" assert turn.tool_calls == [ToolCall("danger", "write", {})] +def _openai_backend_preserves_malformed_arguments(): + """Unparseable arguments are kept as _raw, never silently dropped to an empty call.""" + fake = FakeOpenAIResponses( + [ + { + "model": "gpt", + "status": "completed", + "output": [ + {"type": "function_call", "call_id": "c1", "name": "lookup", "arguments": "{not json"} + ], + "usage": None, + } + ] + ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(fake)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.tool_calls == [ToolCall("c1", "lookup", {"_raw": "{not json"})] + + @pytest.mark.parametrize( "case", case_params( - _openai_backend_translates_tools_calls_and_tool_messages, + _openai_backend_translates_tools_calls_and_outputs, _openai_backend_normalizes_refusal_for_driver_guard, + _openai_backend_preserves_malformed_arguments, ), ) def test_openai_backend_behaviours(case): From 75da834827eef8b00fd04edca997faa5fdba0e5b Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 24 Aug 2026 20:33:58 +0530 Subject: [PATCH 81/93] Record the cause inside an ExceptionGroup, not its sub-exception count A row's error was `f"{type(exc).__name__}: {exc}"`, and an ExceptionGroup's message names only how many sub-exceptions it holds. anyio wraps every driver call in a task group, so that is the normal shape on this path, not an edge case: a plain OpenAI 400 naming the exact unsupported parameter was persisted as "unhandled errors in a TaskGroup (1 sub-exception)", 70 times, and recovering it meant reproducing the call by hand against the live surface. describe_exception flattens nested groups to their leaves and bounds a pathological fan-out. It reads only `.exceptions`, so it needs no version check; the test covers that contract with a stand-in and additionally exercises the real builtin where it exists, since the project still declares Python 3.10 and BaseExceptionGroup is 3.11+. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/errors.py | 30 ++++++++++++++++++++ evals/runner/live.py | 9 ++++-- tests/evals/test_skip_taxonomy.py | 46 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/evals/core/errors.py b/evals/core/errors.py index 21063180..25ac589b 100644 --- a/evals/core/errors.py +++ b/evals/core/errors.py @@ -3,6 +3,36 @@ from __future__ import annotations +def describe_exception(exc: BaseException, *, limit: int = 4) -> str: + """Render an exception for a result row, flattening any ExceptionGroup. + + An ExceptionGroup's own message names only how many sub-exceptions it holds, so recording + ``f"{type(exc).__name__}: {exc}"`` on one throws the diagnosis away: an OpenAI 400 naming + the exact unsupported parameter was persisted as "unhandled errors in a TaskGroup + (1 sub-exception)", and finding it again meant reproducing the call by hand. anyio wraps + everything the driver does in a task group, so this is the normal shape here, not an edge + case. Nested groups are flattened; ``limit`` bounds a pathological fan-out. + """ + leaves: list[str] = [] + + def walk(node: BaseException) -> None: + subs = getattr(node, "exceptions", None) + if subs: + for sub in subs: + if len(leaves) >= limit: + return + walk(sub) + return + leaves.append(f"{type(node).__name__}: {node}") + + walk(exc) + if not leaves: + return f"{type(exc).__name__}: {exc}" + head = f"{type(exc).__name__}: {exc}" if getattr(exc, "exceptions", None) else "" + body = " | ".join(leaves) + return f"{head} -> {body}" if head else body + + class TaskSkipped(Exception): """A task that cannot run in this environment without blaming the agent. diff --git a/evals/runner/live.py b/evals/runner/live.py index 3956a412..871843c4 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -from evals.core.errors import TaskSkipped +from evals.core.errors import TaskSkipped, describe_exception from evals.core.evidence import configured_evidence_labels from evals.core.results import TaskResult, agent_run_to_task_result from evals.core.server_env import stdio_server_env @@ -263,11 +263,14 @@ async def _drive_agent( else: agent_error_class = "infra_cli" row.success = False - row.error = f"{type(exc).__name__}: {exc}" + # Flattened, not str(exc): an ExceptionGroup's message names only its sub-exception + # count, and anyio wraps every driver call in a task group. + described = describe_exception(exc) + row.error = described row.error_class = agent_error_class row.verify_note = "" print( - f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {exc}", + f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {described}", file=sys.stderr, ) return None diff --git a/tests/evals/test_skip_taxonomy.py b/tests/evals/test_skip_taxonomy.py index 474b7526..24efbadb 100644 --- a/tests/evals/test_skip_taxonomy.py +++ b/tests/evals/test_skip_taxonomy.py @@ -62,3 +62,49 @@ def test_task_capability_pairs_are_derived_from_fixture_needs_and_fail_closed(): assert classify_skip_reason("env:plan-gated:work-item-types", task_id="S1") == "expected-capability" assert classify_skip_reason("env:no-activity-worker", task_id="L2") == "expected-capability" assert classify_skip_reason("env:no-activity-worker", task_id="R1") == "unexpected" + + +def test_describe_exception_flattens_a_task_group(): + """The real failure must survive into the row, not the group's sub-exception count. + + An OpenAI 400 naming the exact unsupported parameter was recorded as "unhandled errors in + a TaskGroup (1 sub-exception)", and recovering it meant reproducing the call by hand. + + The helper reads only ``.exceptions``, so the contract is testable on every supported + Python; the genuine builtin is 3.11+, and this project still declares 3.10. + """ + import builtins + import sys + + from evals.core.errors import describe_exception + + class FakeGroup(Exception): + """Anything exposing .exceptions -- which is all the helper looks at.""" + + def __init__(self, message, exceptions): + super().__init__(message) + self.exceptions = tuple(exceptions) + + inner = ValueError("Function tools with reasoning_effort are not supported") + described = describe_exception(FakeGroup("unhandled errors in a TaskGroup", [inner])) + assert "reasoning_effort" in described, "the actual cause was dropped" + assert "ValueError" in described + + # Nested groups flatten to their leaves. + nested = FakeGroup("outer", [FakeGroup("inner", [RuntimeError("deep")])]) + assert "deep" in describe_exception(nested) + + # A plain exception is unchanged in substance. + assert describe_exception(RuntimeError("plain")) == "RuntimeError: plain" + + # A pathological fan-out is bounded rather than unbounded. + many = FakeGroup("many", [RuntimeError(f"e{i}") for i in range(20)]) + assert describe_exception(many, limit=3).count("RuntimeError") == 3 + + # And against the real builtin wherever it exists -- looked up dynamically so this stays + # importable on 3.10 and does not read as an undefined name to the linter. + real_group = getattr(builtins, "BaseExceptionGroup", None) + if real_group is not None and sys.version_info >= (3, 11): + described = describe_exception(real_group("unhandled errors in a TaskGroup", [inner])) + assert "reasoning_effort" in described + assert "sub-exception" not in described.split(" -> ")[-1] From fd09347d53d0fe99384e9fca804b4cc0363259c0 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 24 Aug 2026 21:12:51 +0530 Subject: [PATCH 82/93] Ask Anthropic for prompt caching This loop resends the entire tool surface plus the accumulated transcript on every turn, and the Anthropic backend never asked for caching, so it paid full input price for all of it. The measured OpenAI arm reads 88% of its input from cache, because the Responses API caches unasked where Anthropic's is opt-in -- so the gap was a missing field, not a pricing difference, and a provider cost comparison built on it would have blamed the wrong thing. Priced against the running arm's measured token profile, per 70-row arm: claude-haiku-4-5 $6.98 -> $1.81, claude-sonnet-5 $18.14 -> $4.71. A cache read is 0.1x input and a 5m write 1.25x, so it pays for itself on the second turn of any task. Uses the automatic form -- one top-level field, breakpoint advancing on its own -- rather than per-block breakpoints, since what needs caching here is simply "everything so far" and the conversation grows every turn. Confirmed against the docs and against the installed SDK, which accepts cache_control as a declared parameter of messages.create rather than passing it blind. The test asserts placement at the top level and not on a content block, and fails with the field removed. Co-Authored-By: Claude Opus 5 (1M context) --- evals/drivers/api/anthropic.py | 9 ++++++++ tests/evals/drivers/test_api_driver.py | 29 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py index 448f7ab6..ff144a1c 100644 --- a/evals/drivers/api/anthropic.py +++ b/evals/drivers/api/anthropic.py @@ -85,6 +85,15 @@ def next_turn(self) -> Turn: "max_tokens": self.max_tokens, "messages": self.messages, "tools": self.tools, + # Automatic caching: one top-level field, and the breakpoint advances on its own as + # the conversation grows. Anthropic caching is opt-in where OpenAI's Responses API + # caches unasked, and this loop resends the whole tool surface plus the accumulated + # transcript on every turn -- the measured OpenAI arm reads 88% of its input from + # cache, so without this an Anthropic arm pays full price for the same shape. It also + # made cost differences between the two providers read as pricing rather than as a + # missing field. A cache read is 0.1x input and a 5m write 1.25x, so this pays for + # itself on the second turn of any task. + "cache_control": {"type": "ephemeral"}, } if self.system is not None: request["system"] = self.system diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 57c9f5a5..15b183af 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -704,6 +704,35 @@ def test_openai_backend_derives_stop_reason_from_status(status, incomplete_reaso assert turn.provider_stop_reason == (incomplete_reason or status) +def test_anthropic_backend_requests_automatic_prompt_caching(): + """Every turn resends the whole tool surface plus the transcript, so caching is not optional. + + Anthropic caching is opt-in where OpenAI's Responses API caches unasked. Without this field + an Anthropic arm paid full input price on content the measured OpenAI arm read 88% of from + cache, which made a provider cost comparison read as pricing rather than a missing field. + """ + responses = [ + { + "model": "claude-actual", + "stop_reason": "end_turn", + "content": [{"type": "text", "text": "done"}], + "usage": {"input_tokens": 5, "output_tokens": 1}, + } + ] + messages = FakeAnthropicMessages(responses) + backend = AnthropicBackend("claude", max_tokens=10, client=SimpleNamespace(messages=messages)) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {}}) + backend.start("system", "prompt", [tool]) + + backend.next_turn() + + request = messages.requests[0] + # Top level, not on a content block: that is the automatic form, where the breakpoint + # advances by itself as the conversation grows. + assert request["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in request["messages"][0] + + def _openai_backend_translates_tools_calls_and_outputs(): responses = [ { From 3c786b6e048ac0c6bc40f6262672837e2ab99351 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 24 Aug 2026 21:13:00 +0530 Subject: [PATCH 83/93] Make the agent-loop iteration cap settable MAX_ITERATIONS was a hardcoded 15 that only the API driver enforces: CLI drivers are handed max_turns and discard it -- the antigravity driver says so outright, "no turn-cap flag" -- so their vendor loop decides. A cap that binds therefore penalises API arms against CLI arms of the same model rather than limiting both, and 15 does bind: codex spent 18 calls on W6 in a CLI arm, so tasks in this battery legitimately need more. Observed: an API arm failed R2 and R3 at exactly 16 calls with hit_max_iterations, truncated rather than wrong, while the paired CLI arm passed both. Raising the cap cleared them. Default stays 15 so no existing run changes; --max-iterations lifts it for a cross-driver comparison and records in the command that it did. Co-Authored-By: Claude Opus 5 (1M context) --- evals/cli.py | 11 +++++++++++ evals/runner/live.py | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/evals/cli.py b/evals/cli.py index b3afc71f..900df960 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -148,6 +148,16 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "(off by default; sidecars and rows may contain live workspace data)" ), ) + p.add_argument( + "--max-iterations", + type=int, + default=None, + help=( + "API driver only: cap on agent loop iterations (default 15). CLI drivers discard " + "this and let their own loop decide, so a cap that binds penalises only the API " + "side; raise it when comparing an API arm against a CLI arm of the same model." + ), + ) p.add_argument("--out", type=str, default=None, help="JSONL output path") p.add_argument( "--resume", @@ -307,6 +317,7 @@ def main(argv: list[str] | None = None) -> int: server_env=server_env or None, resume=bool(args.resume), record_result_payloads=bool(args.record_result_payloads), + **({"max_iterations": args.max_iterations} if args.max_iterations else {}), resolved_model_id=model_id, ) ) diff --git a/evals/runner/live.py b/evals/runner/live.py index 871843c4..564da9a8 100644 --- a/evals/runner/live.py +++ b/evals/runner/live.py @@ -30,6 +30,12 @@ from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision from .resume import load_resume_skip_keys +# Only the API driver enforces this: CLI drivers are handed max_turns and discard it (the +# antigravity driver says so outright -- "no turn-cap flag"), so their vendor loop decides. +# A cap that binds therefore biases API arms against CLI arms of the same model rather than +# limiting both, and 15 does bind: codex spent 18 calls on W6 in a CLI arm, so tasks in this +# battery legitimately need more. Kept as the default so no existing run changes, and made +# settable so a cross-driver comparison can lift it out of the way and say it did. MAX_ITERATIONS = 15 MAX_TOKENS = 8192 @@ -89,6 +95,7 @@ async def run_agent_task_via_driver( workspace_slug: str, server_env: dict[str, str] | None = None, artifact_dir: Path | None = None, + max_iterations: int = MAX_ITERATIONS, ) -> TaskResult: """Run one task through the selected driver.""" project_name = ctx["project_name"] @@ -102,7 +109,7 @@ async def run_agent_task_via_driver( prompt, mcp_env, model_id, - MAX_ITERATIONS, + max_iterations, system=system, cwd=Path(__file__).resolve().parent.parent.parent, evidence_sentinels=ctx.get("evidence_sentinels"), @@ -227,6 +234,7 @@ async def _drive_agent( repetition: int, is_api_driver: bool, artifact_dir: Path, + max_iterations: int = MAX_ITERATIONS, ) -> TaskResult | None: """Run the agent and classify launch or prompt failures.""" # Agent wrap: API failures and CLI failures are infrastructure. @@ -240,6 +248,7 @@ async def _drive_agent( workspace_slug=workspace_slug, server_env=server_env, artifact_dir=artifact_dir, + max_iterations=max_iterations, ) except PromptBindError as exc: # Empty/missing seed IDs in the prompt — not an agent failure. @@ -482,6 +491,7 @@ async def _run_task_repetition( external: bool, server_env: dict[str, str] | None, artifact_dir: Path, + max_iterations: int = MAX_ITERATIONS, ) -> TaskResult: """Seed, drive, verify, assemble, and remove one task repetition.""" context: dict[str, Any] = {} @@ -513,6 +523,7 @@ async def _run_task_repetition( repetition=repetition, is_api_driver=is_api_driver, artifact_dir=artifact_dir, + max_iterations=max_iterations, ) if agent is not None: _apply_agent_run( @@ -565,6 +576,7 @@ async def run_live( resume: bool = False, record_result_payloads: bool = False, resolved_model_id: str | None = None, + max_iterations: int = MAX_ITERATIONS, ) -> int: label = (label or "local").strip() or "local" external = server_cmd is not None @@ -688,6 +700,7 @@ async def run_live( external=external, server_env=server_env, artifact_dir=artifact_dir, + max_iterations=max_iterations, ) file.write(json.dumps(row.to_row(), default=str) + "\n") file.flush() From f2a41b615798f8f406b826b10b58bbf90aa7ee11 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Mon, 24 Aug 2026 23:55:17 +0530 Subject: [PATCH 84/93] feat(evals): normalise token accounting and price runs usage_total.input_tokens does not mean the same thing across drivers: it is inclusive of cache under OpenAI Responses and exclusive under Anthropic Messages and every CLI vendor. Reading it naively misprices by ~4x, which is how two conclusions about relative cost reversed. Driver family cannot decide this -- the same api driver runs both providers and both arrive as source "iterations" with opposite meanings. So backends now declare input_tokens_include_cache and the driver records cache_semantics on every run; token_accounting resolves declared > explicit total > no-cache > model family, and refuses rather than guessing when none apply. pricing keeps three outcomes distinct. The failure mode designed against is not an absent number but a zero: antigravity records no usage on any row, and pricing that as $0.00 reads as free rather than as unmeasured. Cross-checking computed cost against Claude Code's reported total_cost_usd caught a wrong assumption on the first run: published 5-minute cache-write rates priced a measured arm at $2.46 against a reported $3.18, and the 1-hour TTL multiplier reproduces it exactly. Verified against the 2026-08-24 arms: api $0.502, codex-cli $3.238, claude-cli computed == vendor $3.181, agy unmeasured on 70/70 rows. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/pricing.py | 150 ++++++++++++++++++++++++ evals/core/token_accounting.py | 163 +++++++++++++++++++++++++++ evals/drivers/api/anthropic.py | 2 + evals/drivers/api/base.py | 6 + evals/drivers/api/driver.py | 6 + evals/drivers/api/openai.py | 2 + tests/evals/test_pricing.py | 99 ++++++++++++++++ tests/evals/test_token_accounting.py | 134 ++++++++++++++++++++++ 8 files changed, 562 insertions(+) create mode 100644 evals/core/pricing.py create mode 100644 evals/core/token_accounting.py create mode 100644 tests/evals/test_pricing.py create mode 100644 tests/evals/test_token_accounting.py diff --git a/evals/core/pricing.py b/evals/core/pricing.py new file mode 100644 index 00000000..d557483c --- /dev/null +++ b/evals/core/pricing.py @@ -0,0 +1,150 @@ +"""What a run cost, or an honest statement that we do not know. + +Cost is the decision metric for a tool-surface project, and for two days nothing +computed it -- so it was derived by hand and stated wrongly twice. The failure mode +this module is built against is not an absent number but a **zero**: an arm with no +usage recorded, priced at $0.00, reads as free rather than as unmeasured. So there +are three outcomes and they are kept distinct: + + priced usage present, model known + unpriced usage present, but the model is not in the table, or its cache + semantics could not be resolved + unmeasured the driver recorded no usage at all -- true of every antigravity row + +Prices go stale, and a wrong price is worse than no price. ``PRICES_AS_OF`` dates the +table, but a date detects nothing on its own. The mechanism that actually detects +staleness is ``vendor_usd``: Claude Code reports ``total_cost_usd`` per run, so the +computed figure can be checked against the vendor's own on every run. That check +earned its keep immediately -- published 5-minute cache-write rates priced a measured +arm at $2.46 against a reported $3.18, and the 1-hour TTL multiplier reproduced the +reported figure exactly. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from evals.core.token_accounting import normalize_usage + +#: The day the rates below were last checked against published pricing. +PRICES_AS_OF = "2026-08-24" + +PRICED = "priced" +UNPRICED = "unpriced" +UNMEASURED = "unmeasured" + +COST_OUTCOMES = (PRICED, UNPRICED, UNMEASURED) + + +@dataclass(frozen=True) +class ModelPrice: + """US dollars per million tokens. + + ``cache_creation`` is the rate for tokens *written* to cache. Anthropic charges + for writes on a multiple of the input rate that depends on the cache TTL -- 1.25x + at five minutes, 2x at one hour -- and Claude Code uses the one-hour tier, which + is what reproduces its reported cost. OpenAI does not bill cache writes at all + and reports zero such tokens, so the value is unexercised there. + """ + + input: float + cached_input: float + output: float + cache_creation: float | None = None + + def cache_creation_rate(self) -> float: + return self.input if self.cache_creation is None else self.cache_creation + + +#: Keyed by model family prefix, longest match first. Dated ids such as +#: ``claude-haiku-4-5-20251001`` match their undated family. +PRICES: dict[str, ModelPrice] = { + "gpt-5.6-luna": ModelPrice(input=0.20, cached_input=0.02, output=1.20), + "claude-haiku-4-5": ModelPrice(input=1.00, cached_input=0.10, output=5.00, cache_creation=2.00), + "claude-sonnet-5": ModelPrice(input=3.00, cached_input=0.30, output=15.00, cache_creation=6.00), + "claude-opus-5": ModelPrice(input=5.00, cached_input=0.50, output=25.00, cache_creation=10.00), +} + + +@dataclass(frozen=True) +class RowCost: + """The cost of one row, and how confident that figure is.""" + + outcome: str + usd: float | None + model_id: str | None + vendor_usd: float | None = None + + @property + def billed_usd(self) -> float | None: + """The vendor's own figure where it exists, else ours. + + A provider that reports what it charged is more authoritative than any table. + """ + return self.vendor_usd if self.vendor_usd is not None else self.usd + + +def resolve_model_id(usage_total: Mapping[str, Any] | None, *, model: str | None) -> str | None: + """Return the most specific model identifier this row carries. + + ``row.model`` is what the driver was asked for, which under a CLI is a tier alias + -- claude-cli records ``"haiku"``. The real identifier is inside ``modelUsage``, + so that wins when a single model produced the run. + """ + if usage_total: + model_usage = usage_total.get("modelUsage") + if isinstance(model_usage, Mapping) and len(model_usage) == 1: + only = next(iter(model_usage)) + if isinstance(only, str) and only: + return only + return model or None + + +def lookup_price(model_id: str | None) -> ModelPrice | None: + if not model_id: + return None + lowered = model_id.strip().lower() + for prefix in sorted(PRICES, key=len, reverse=True): + if lowered.startswith(prefix): + return PRICES[prefix] + return None + + +def price_usage(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> RowCost: + """Price one row's ``usage_total``.""" + if not usage_total: + return RowCost(outcome=UNMEASURED, usd=None, model_id=model or None) + + vendor = usage_total.get("total_cost_usd") + vendor_usd = float(vendor) if isinstance(vendor, (int, float)) else None + model_id = resolve_model_id(usage_total, model=model) + + accounting = normalize_usage(usage_total, model=model_id) + price = lookup_price(model_id) + if accounting is None or price is None: + return RowCost(outcome=UNPRICED, usd=None, model_id=model_id, vendor_usd=vendor_usd) + + usd = ( + accounting.uncached_input * price.input + + accounting.cached_input * price.cached_input + + accounting.cache_creation * price.cache_creation_rate() + + accounting.output * price.output + ) / 1e6 + return RowCost(outcome=PRICED, usd=round(usd, 10), model_id=model_id, vendor_usd=vendor_usd) + + +__all__ = [ + "COST_OUTCOMES", + "PRICED", + "PRICES", + "PRICES_AS_OF", + "UNMEASURED", + "UNPRICED", + "ModelPrice", + "RowCost", + "lookup_price", + "price_usage", + "resolve_model_id", +] diff --git a/evals/core/token_accounting.py b/evals/core/token_accounting.py new file mode 100644 index 00000000..3f687051 --- /dev/null +++ b/evals/core/token_accounting.py @@ -0,0 +1,163 @@ +"""One reading of ``usage_total``, whatever driver produced it. + +``usage_total.input_tokens`` does not mean the same thing everywhere, and reading it +naively misprices by roughly 4x on one side of any cross-driver comparison: + + inclusive the field already contains the cached reads (OpenAI Responses). The + uncached portion is the remainder. + exclusive the field is net of cache, and the cached reads sit beside it + (Anthropic Messages, and every CLI vendor measured). + +Driver family cannot decide this. The same api driver runs both providers, so an +Anthropic arm and an OpenAI arm arrive with identical ``source: "iterations"`` and +opposite meanings. What decides it, in order of authority: + + declared the driver recorded ``cache_semantics`` outright. Always trusted. + explicit_total ``total_input_tokens_including_cache`` is present, which only an + exclusive shape carries, and it states the total directly. + no_cache nothing was cached, so both readings coincide and no guess is + needed -- this covers unknown models safely. + model_family inferred from the model name, the last resort for rows recorded + before ``cache_semantics`` existed. + +When none of those apply the answer is ``None``. Refusing is deliberate: a guess here +is invisible in the output and wrong by a factor that reverses conclusions. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +INCLUSIVE = "inclusive" +EXCLUSIVE = "exclusive" + +#: Substrings that identify a provider's cache convention from a model name. +#: Only families whose semantics have been verified appear here; an unmatched name +#: with cache activity yields ``None`` rather than a default. +_MODEL_FAMILIES: tuple[tuple[tuple[str, ...], str], ...] = ( + (("claude", "anthropic"), EXCLUSIVE), + (("gpt", "o1-", "o3-", "o4-", "openai"), INCLUSIVE), +) + + +@dataclass(frozen=True) +class TokenAccounting: + """Input split into what was paid for fresh, read from cache, and written to it. + + ``uncached_input + cached_input + cache_creation == total_input`` always holds, so + a caller can price the three parts at three rates without knowing the source shape. + """ + + uncached_input: int + cached_input: int + cache_creation: int + output: int + total_input: int + semantics: str + semantics_source: str + + +def _int(usage: Mapping[str, Any], name: str) -> int: + try: + return max(0, int(usage.get(name) or 0)) + except (TypeError, ValueError): + return 0 + + +def _family_semantics(model: str | None) -> str | None: + lowered = (model or "").strip().lower() + if not lowered: + return None + for markers, semantics in _MODEL_FAMILIES: + if any(marker in lowered for marker in markers): + return semantics + return None + + +def cache_semantics_of(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> str | None: + """Return how this row's ``input_tokens`` treats cache, or None if undecidable.""" + if not usage_total: + return None + declared = usage_total.get("cache_semantics") + if declared in (INCLUSIVE, EXCLUSIVE): + return str(declared) + if usage_total.get("total_input_tokens_including_cache") is not None: + return EXCLUSIVE + if _int(usage_total, "cache_read_input_tokens") + _int(usage_total, "cache_creation_input_tokens") == 0: + # Both readings agree when nothing was cached. + return INCLUSIVE + return _family_semantics(model) + + +def normalize_usage( + usage_total: Mapping[str, Any] | None, + *, + model: str | None = None, +) -> TokenAccounting | None: + """Normalise one row's usage, or None when it is absent or undecidable. + + A caller distinguishes the two None cases by the input: a falsy ``usage_total`` + means the driver recorded no usage at all (report it as unmeasured), while a + populated one means the shape could not be read (report it as unpriced). + """ + if not usage_total: + return None + + cached = _int(usage_total, "cache_read_input_tokens") + creation = _int(usage_total, "cache_creation_input_tokens") + output = _int(usage_total, "output_tokens") + reported_input = _int(usage_total, "input_tokens") + + declared = usage_total.get("cache_semantics") + explicit_total = usage_total.get("total_input_tokens_including_cache") + + if declared in (INCLUSIVE, EXCLUSIVE): + semantics, source = str(declared), "declared" + elif explicit_total is not None: + semantics, source = EXCLUSIVE, "explicit_total" + elif cached + creation == 0: + semantics, source = INCLUSIVE, "no_cache" + else: + inferred = _family_semantics(model) + if inferred is None: + return None + semantics, source = inferred, "model_family" + + if semantics == EXCLUSIVE: + uncached = reported_input + total = uncached + cached + creation + else: + total = reported_input + uncached = total - cached - creation + + if source == "explicit_total": + # The vendor states the total as well as the parts. Both agreeing is what makes + # this shape self-validating; disagreement means the shape changed underneath us, + # and an unpriced row is a visible failure where a wrong price is not. + if total != _int(usage_total, "total_input_tokens_including_cache"): + return None + + if uncached < 0: + # An inclusive reading whose cache exceeds its total is not a reading at all. + return None + + return TokenAccounting( + uncached_input=uncached, + cached_input=cached, + cache_creation=creation, + output=output, + total_input=total, + semantics=semantics, + semantics_source=source, + ) + + +__all__ = [ + "EXCLUSIVE", + "INCLUSIVE", + "TokenAccounting", + "cache_semantics_of", + "normalize_usage", +] diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py index ff144a1c..2d7e76f7 100644 --- a/evals/drivers/api/anthropic.py +++ b/evals/drivers/api/anthropic.py @@ -49,6 +49,8 @@ class AnthropicBackend: """Stateful adapter over stable ``client.messages.create`` calls.""" provider = "anthropic" + # Messages reports input_tokens net of both cache fields, so the three add up. + input_tokens_include_cache = False def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: if client is None: diff --git a/evals/drivers/api/base.py b/evals/drivers/api/base.py index d512a4f3..2b9261e7 100644 --- a/evals/drivers/api/base.py +++ b/evals/drivers/api/base.py @@ -84,6 +84,12 @@ class ModelBackend(Protocol): actual_model: str client: Any + #: Whether this provider's ``usage.input_tokens`` already contains cached reads. + #: The two providers disagree, and the driver records the answer on every run so + #: cost analysis never has to infer it from a model name. See + #: ``evals.core.token_accounting``. + input_tokens_include_cache: bool + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: ... def next_turn(self) -> Turn: ... diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 772e86e4..592d0c04 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -34,6 +34,7 @@ observed_sentinel_labels, ) from evals.core.results import AgentRun, Usage +from evals.core.token_accounting import EXCLUSIVE, INCLUSIVE from evals.core.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens from evals.core.tool_manifest import ToolManifestCapture, tools_page from evals.drivers.api.base import ( @@ -415,6 +416,11 @@ async def _run_task( "cache_read_input_tokens": total_cache_read_input_tokens, "cache_creation_input_tokens": total_cache_creation_input_tokens, "source": "iterations", + # Anthropic and OpenAI disagree about whether input_tokens already contains + # the cached reads, and both arrive here as source "iterations". Recording + # which one this was is the difference between pricing a run and guessing + # at it from the model name. + "cache_semantics": (INCLUSIVE if getattr(backend, "input_tokens_include_cache", False) else EXCLUSIVE), } if manifest_state["stale"]: tool_manifest_fingerprint = None diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py index c60a37c2..4adda097 100644 --- a/evals/drivers/api/openai.py +++ b/evals/drivers/api/openai.py @@ -94,6 +94,8 @@ class OpenAIBackend: """ provider = "openai" + # Responses counts cached reads inside input_tokens; cached_tokens is a subset of it. + input_tokens_include_cache = True def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: if client is None: diff --git a/tests/evals/test_pricing.py b/tests/evals/test_pricing.py new file mode 100644 index 00000000..0211a1cb --- /dev/null +++ b/tests/evals/test_pricing.py @@ -0,0 +1,99 @@ +"""Pricing must be right, refuse, or say it has nothing -- never quietly zero.""" + +from __future__ import annotations + +from evals.core.pricing import ( + PRICED, + PRICES_AS_OF, + UNMEASURED, + UNPRICED, + price_usage, + resolve_model_id, +) + +OPENAI_ROW = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", +} +CLAUDE_CLI_ROW = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + "source": "modelUsage", +} + + +def test_prices_carry_an_as_of_date(): + assert PRICES_AS_OF + + +def test_a_known_model_is_priced(): + cost = price_usage(OPENAI_ROW, model="gpt-5.6-luna") + assert cost.outcome == PRICED + assert cost.usd is not None + # 1.27M fresh at $0.20, 83460 cached at $0.02, 1121 out at $1.20. + assert cost.usd == round((14353 * 0.20 + 83460 * 0.02 + 1121 * 1.20) / 1e6, 10) + + +def test_an_unknown_model_is_unpriced_not_free(): + """A silent zero reads as 'free'. The whole point is that it must read as 'unknown'.""" + cost = price_usage(OPENAI_ROW, model="some-model-nobody-has-heard-of") + assert cost.outcome == UNPRICED + assert cost.usd is None + + +def test_a_row_with_no_usage_at_all_is_unmeasured(): + """antigravity records usage_total on 0 of 70 rows; that is not the same as unknown.""" + for empty in (None, {}): + cost = price_usage(empty, model="gemini-3.6-flash-low") + assert cost.outcome == UNMEASURED + assert cost.usd is None + + +def test_unmeasured_and_unpriced_are_distinguishable(): + assert UNMEASURED != UNPRICED + assert price_usage(None, model="gpt-5.6-luna").outcome != price_usage(OPENAI_ROW, model="nope").outcome + + +def test_model_id_resolves_through_model_usage_before_the_row_alias(): + """claude-cli records model 'haiku' -- an alias, unpriceable as a key.""" + assert resolve_model_id(CLAUDE_CLI_ROW, model="haiku") == "claude-haiku-4-5-20251001" + assert resolve_model_id(OPENAI_ROW, model="gpt-5.6-luna") == "gpt-5.6-luna" + assert resolve_model_id(None, model="haiku") == "haiku" + + +def test_dated_model_ids_match_their_family(): + assert price_usage(CLAUDE_CLI_ROW, model="haiku").outcome == PRICED + + +def test_computed_cost_agrees_with_the_vendor_reported_cost(): + """The only mechanism that detects a stale price table. + + Published 5-minute cache-write rates give $2.46 against this arm's reported + $3.18; the 1-hour TTL multiplier reproduces it. Without this check the table + would have shipped 23% low and looked fine. + """ + cost = price_usage(CLAUDE_CLI_ROW, model="haiku") + assert cost.vendor_usd == 0.0753878 + assert cost.usd is not None + assert abs(cost.usd - cost.vendor_usd) / cost.vendor_usd < 0.01 + + +def test_vendor_cost_is_preferred_when_present(): + cost = price_usage(CLAUDE_CLI_ROW, model="haiku") + assert cost.billed_usd == cost.vendor_usd + # ...and falls back to the computed figure when the vendor reports nothing. + assert price_usage(OPENAI_ROW, model="gpt-5.6-luna").billed_usd == price_usage(OPENAI_ROW, model="gpt-5.6-luna").usd + + +def test_undecidable_cache_semantics_is_unpriced(): + """A cached row whose model family is unknown cannot be normalised, so it cannot be priced.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 400} + assert price_usage(row, model="mystery-model").outcome == UNPRICED diff --git a/tests/evals/test_token_accounting.py b/tests/evals/test_token_accounting.py new file mode 100644 index 00000000..a90de83a --- /dev/null +++ b/tests/evals/test_token_accounting.py @@ -0,0 +1,134 @@ +"""Both cache semantics must normalise to the same meaning.""" + +from __future__ import annotations + +import pytest + +from evals.core.token_accounting import ( + EXCLUSIVE, + INCLUSIVE, + TokenAccounting, + cache_semantics_of, + normalize_usage, +) + +# Shapes taken verbatim from real arms (2026-08-24 battery eaf35e8019aa). +OPENAI_ROW = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", +} +CODEX_CLI_ROW = { + "input_tokens": 249983, + "output_tokens": 682, + "cache_read_input_tokens": 224768, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 474751, + "source": "codex_token_count", +} +CLAUDE_CLI_ROW = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "source": "modelUsage", +} + + +def test_explicit_total_is_treated_as_exclusive(): + """A recorded total means input_tokens excludes cache; the total is authoritative.""" + accounting = normalize_usage(CODEX_CLI_ROW) + assert accounting == TokenAccounting( + uncached_input=249983, + cached_input=224768, + cache_creation=0, + output=682, + total_input=474751, + semantics=EXCLUSIVE, + semantics_source="explicit_total", + ) + + +def test_openai_input_tokens_are_inclusive_of_cache(): + """Responses counts cached reads inside input_tokens; uncached is the remainder.""" + accounting = normalize_usage(OPENAI_ROW, model="gpt-5.6-luna") + assert accounting is not None + assert accounting.semantics == INCLUSIVE + assert accounting.total_input == 97813 + assert accounting.uncached_input == 97813 - 83460 + + +def test_anthropic_api_input_tokens_are_exclusive_of_cache(): + """The Messages API reports input_tokens net of both cache fields. + + The same api driver produces this and the OpenAI shape above, so driver family + cannot decide the semantics -- this is the case the first plan draft got wrong. + """ + row = { + "input_tokens": 1200, + "output_tokens": 300, + "cache_read_input_tokens": 50000, + "cache_creation_input_tokens": 2000, + "source": "iterations", + } + accounting = normalize_usage(row, model="claude-haiku-4-5") + assert accounting is not None + assert accounting.semantics == EXCLUSIVE + assert accounting.uncached_input == 1200 + assert accounting.total_input == 1200 + 50000 + 2000 + + +def test_declared_semantics_beat_inference(): + """A driver that records what it means is trusted over any model-name guess.""" + row = dict(OPENAI_ROW, cache_semantics=EXCLUSIVE) + accounting = normalize_usage(row, model="gpt-5.6-luna") + assert accounting is not None + assert accounting.semantics == EXCLUSIVE + assert accounting.semantics_source == "declared" + assert accounting.total_input == 97813 + 83460 + + +def test_uncached_row_needs_no_semantics_at_all(): + """With no cache activity the two readings coincide, so an unknown model is fine.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 0} + accounting = normalize_usage(row, model="some-model-nobody-has-heard-of") + assert accounting is not None + assert accounting.semantics_source == "no_cache" + assert accounting.total_input == 500 + assert accounting.uncached_input == 500 + + +def test_cached_row_with_unknown_model_refuses_to_guess(): + """Guessing here misprices by ~4x, which is how two conclusions reversed.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 400} + assert normalize_usage(row, model="some-model-nobody-has-heard-of") is None + assert cache_semantics_of(row, model=None) is None + + +def test_identity_violation_is_not_silently_priced(): + """input + cache_read + cache_creation == total holds 70/70 on both CLI vendors. + + If a vendor changes shape the sum stops matching, and reporting the row as + unpriced is the loud outcome; using either number would misprice invisibly. + """ + broken = dict(CODEX_CLI_ROW, total_input_tokens_including_cache=999999) + assert normalize_usage(broken) is None + + +def test_absent_usage_is_none(): + assert normalize_usage(None) is None + assert normalize_usage({}) is None + + +@pytest.mark.parametrize("row", [OPENAI_ROW, CODEX_CLI_ROW, CLAUDE_CLI_ROW]) +def test_parts_never_exceed_the_total(row): + """Whatever the shape, the normalised parts must reconstruct the total input.""" + model = "gpt-5.6-luna" if row is OPENAI_ROW else None + accounting = normalize_usage(row, model=model) + assert accounting is not None + assert accounting.uncached_input + accounting.cached_input + accounting.cache_creation == accounting.total_input + assert accounting.uncached_input >= 0 From ac2fcfc387881a2a0abc3390ec45c51abfc5017a Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:00:49 +0530 Subject: [PATCH 85/93] feat(evals): report cost, input, result volume and latency in the A/B view Every number here was already recorded, and result tokens already reached --table. None of it reached the two-file A/B view, which is the view a two-arm question is actually asked in. That placement is why the inversion went unseen: one view had the metric, the other had the statistics. The A/B block now prints paired deltas for input tokens, result tokens, cost, wall time and call latency in the same shape as the existing call delta, then an economics line per arm. On the 2026-08-24 pair the call delta (-1.5, B does less work) now sits directly above the cost delta (+$0.037, CI excluding zero, B costs more) -- the two facts that pointed opposite ways. Arm totals cover every executed row, because cost was incurred whether or not the task passed; per-task values use the successful trace-intact rows the call delta already uses, so a paired delta compares like with like. An arm whose driver recorded no usage prints "unmeasured" rather than a figure. Verified on the real arms: A $0.502 / 8,659,147 input; B $3.238 / 27,139,602. Worth noting the result-token CI straddles zero, so that gap -- unlike input and cost -- is not significant. Co-Authored-By: Claude Opus 5 (1M context) --- evals/report/compare.py | 97 ++++++++++++++ evals/report/economics.py | 194 +++++++++++++++++++++++++++ evals/report/summary.py | 3 + tests/evals/report/test_economics.py | 93 +++++++++++++ 4 files changed, 387 insertions(+) create mode 100644 evals/report/economics.py create mode 100644 tests/evals/report/test_economics.py diff --git a/evals/report/compare.py b/evals/report/compare.py index aae635ba..32a21618 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any +from .economics import economics_statement from .load import ResultRow, RunKeyValidation from .off_surface import off_surface_statement from .schema_friction import measure_schema_friction, schema_friction_statement, successful_trace_rows @@ -13,6 +14,36 @@ from .table import format_number +def _paired_metric( + economics_a: Any, + economics_b: Any, + shared: list[str], + attribute: str, +) -> dict[str, Any]: + """Pair one per-task resource metric across arms, skipping tasks either side lacks. + + Same shape as the call delta -- mean, median and a paired bootstrap -- so the + resource lines read against it directly rather than in different units. + """ + deltas: list[float] = [] + for task_id in shared: + task_a = economics_a.tasks.get(task_id) + task_b = economics_b.tasks.get(task_id) + if task_a is None or task_b is None: + continue + value_a = getattr(task_a, attribute) + value_b = getattr(task_b, attribute) + if value_a is None or value_b is None: + continue + deltas.append(float(value_b) - float(value_a)) + return { + "n": len(deltas), + "mean_delta": sum(deltas) / len(deltas) if deltas else None, + "median_delta": median(deltas), + "ci": paired_bootstrap_mean_ci(deltas), + } + + def ab_compare( rows_a: list[ResultRow], rows_b: list[ResultRow], @@ -104,10 +135,32 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: } ) + # Resource deltas over the same shared tasks as the call delta. Fewer calls and + # less spend are different virtues, and reporting one without the other is what + # let a 32%-fewer-calls arm read as the cheaper one while burning 3.1x the input. + economics_a, economics_b = summary_a.economics, summary_b.economics + paired_resources = { + name: _paired_metric(economics_a, economics_b, shared, attribute) + for name, attribute in ( + ("input", "med_total_input"), + ("result_tokens", "med_result_tokens"), + ("wall_time", "med_wall_time_s"), + ("call_latency", "med_call_latency_ms"), + ("cost", "cost_usd"), + ) + } + return { "summary_a": summary_a, "summary_b": summary_b, "paired_tasks": per_task, + "economics_a": economics_a, + "economics_b": economics_b, + "total_input_a": economics_a.total_input_tokens, + "total_input_b": economics_b.total_input_tokens, + "cost_a": economics_a.cost_usd, + "cost_b": economics_b.cost_usd, + "paired_resources": paired_resources, "mean_delta": sum(deltas) / len(deltas) if deltas else None, "median_delta": median(deltas), "call_permutation_p": paired_permutation_pvalue(deltas), @@ -153,6 +206,49 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: } +#: Label, units and precision for each paired resource delta. +_RESOURCE_LINES: tuple[tuple[str, str, str, int], ...] = ( + ("input", "input tokens", "", 0), + ("result_tokens", "result tokens", "", 0), + ("cost", "cost", "$", 4), + ("wall_time", "wall time", "s", 1), + ("call_latency", "call latency", "ms", 0), +) + + +def _print_resource_deltas(comparison: dict[str, Any]) -> None: + """Print the resource deltas beside the call delta, in the same paired shape. + + These sit immediately after the call delta on purpose: the call delta alone says + which arm did less work, which is not the same question as which arm cost less, + and the two answers pointed opposite ways on the run that motivated this. + """ + paired = comparison.get("paired_resources") or {} + for key, label, unit, places in _RESOURCE_LINES: + metric = paired.get(key) + if not metric or not metric["n"]: + print(f" median {label} delta (B−A): n/a (no paired tasks reporting it)") + continue + low, high = metric["ci"] + prefix = unit if unit == "$" else "" + suffix = "" if unit == "$" else unit + interval = ( + f" paired-bootstrap95 [{prefix}{low:+,.{places}f}{suffix},{prefix}{high:+,.{places}f}{suffix}]" + if low is not None and high is not None + else "" + ) + print( + f" median {label} delta (B−A): {prefix}{metric['median_delta']:+,.{places}f}{suffix}" + f"{interval} (n={metric['n']} tasks)" + ) + for label, key in (("A", "economics_a"), ("B", "economics_b")): + economics = comparison.get(key) + if economics is None: + continue + for line in economics_statement(economics).splitlines(): + print(f" {label} {line}") + + def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> None: print(f"A/B compare: A={path_a} B={path_b}") success_a, success_b = comparison["success_a"], comparison["success_b"] @@ -222,6 +318,7 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N f"paired-bootstrap95 [{rate_lo * 100:+.1f},{rate_hi * 100:+.1f}] " f"(n={comparison['n_paired_errored_call_rates']} tasks)" ) + _print_resource_deltas(comparison) multiple_repetitions = bool(comparison.get("multi_rep")) if comparison["paired_tasks"]: print() diff --git a/evals/report/economics.py b/evals/report/economics.py new file mode 100644 index 00000000..b281fb65 --- /dev/null +++ b/evals/report/economics.py @@ -0,0 +1,194 @@ +"""What a run cost and how much it moved, in the view where two arms are compared. + +Every number here was already recorded. Result tokens even reached ``--table``. None +of it reached the two-file A/B view, which is where a two-arm question is actually +asked -- so an arm making 32% fewer tool calls while burning 3.1x the input tokens +read as the efficient one until someone totalled the tokens by hand. + +Two populations, deliberately: + + arm totals every executed row, because cost was incurred whether or not the + task passed. + per task the successful, trace-intact rows that call deltas already use, so a + paired delta compares like with like. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass + +from evals.core.pricing import PRICED, PRICES_AS_OF, UNMEASURED, UNPRICED, price_usage +from evals.core.results import TaskResult +from evals.core.token_accounting import normalize_usage + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .schema_friction import successful_trace_rows +from .statistics import median, percentile + +COST_LIMITATION = ( + "limitation: cost is computed from a static price table; a model absent from it reports " + "unpriced, and a row whose driver recorded no usage at all reports unmeasured. Neither is $0" +) + + +@dataclass(frozen=True, slots=True) +class TaskEconomics: + """One task's resource footprint across its eligible repetitions.""" + + task_id: str + repetitions: int + med_total_input: float | None + med_result_tokens: float | None + med_wall_time_s: float | None + med_call_latency_ms: float | None + cost_usd: float | None + + +@dataclass(frozen=True, slots=True) +class EconomicsMeasurement: + """Arm-level totals plus the per-task values a paired delta needs.""" + + tasks: dict[str, TaskEconomics] + total_input_tokens: int | None + total_result_tokens: int + total_wall_time_s: float + cost_usd: float | None + vendor_cost_usd: float | None + cost_outcome: str + priced_rows: int + unpriced_rows: int + unmeasured_rows: int + med_call_latency_ms: float | None + p95_call_latency_ms: float | None + prices_as_of: str = PRICES_AS_OF + + @property + def cost_text(self) -> str: + """Never render an unknown cost as a number.""" + if self.cost_outcome == UNMEASURED or self.cost_usd is None: + return UNMEASURED if self.cost_outcome == UNMEASURED else UNPRICED + text = f"${self.cost_usd:,.3f}" + if self.unpriced_rows or self.unmeasured_rows: + text += f" (+{self.unpriced_rows} unpriced, {self.unmeasured_rows} unmeasured rows)" + return text + + +def _executed_rows(rows: list[ResultRow]) -> list[TaskResult]: + executed: list[TaskResult] = [] + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + continue + executed.append(row) + return executed + + +def _row_input_tokens(row: TaskResult) -> int | None: + accounting = normalize_usage(row.usage_total, model=row.model) + return accounting.total_input if accounting else None + + +def _call_latencies(rows: list[TaskResult]) -> list[float]: + return [float(call.duration_ms) for row in rows for call in row.calls if call.duration_ms is not None] + + +def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: + """Total an arm's cost and volume, and break it down per task.""" + executed = _executed_rows(rows) + + total_input = 0 + saw_input = False + cost_total = 0.0 + vendor_total = 0.0 + saw_vendor = False + priced = unpriced = unmeasured = 0 + for row in executed: + tokens = _row_input_tokens(row) + if tokens is not None: + total_input += tokens + saw_input = True + cost = price_usage(row.usage_total, model=row.model) + if cost.outcome == PRICED: + priced += 1 + if cost.billed_usd is not None: + cost_total += cost.billed_usd + elif cost.outcome == UNPRICED: + unpriced += 1 + else: + unmeasured += 1 + if cost.vendor_usd is not None: + vendor_total += cost.vendor_usd + saw_vendor = True + + if priced == 0: + # No priced row at all: say which kind of nothing this is. + outcome = UNMEASURED if unpriced == 0 else UNPRICED + elif unpriced or unmeasured: + outcome = PRICED # partial, and the counts are carried alongside + else: + outcome = PRICED + + by_task: dict[str, list[TaskResult]] = defaultdict(list) + for row in successful_trace_rows(rows): + by_task[row.task_id].append(row) + + tasks: dict[str, TaskEconomics] = {} + for task_id in sorted(by_task): + task_rows = by_task[task_id] + inputs = [float(value) for value in (_row_input_tokens(row) for row in task_rows) if value is not None] + costs = [price_usage(row.usage_total, model=row.model).billed_usd for row in task_rows] + known_costs = [value for value in costs if value is not None] + tasks[task_id] = TaskEconomics( + task_id=task_id, + repetitions=len(task_rows), + med_total_input=median(inputs), + med_result_tokens=median([float(row.total_result_tokens) for row in task_rows]), + med_wall_time_s=median([float(row.wall_time_s) for row in task_rows]), + med_call_latency_ms=median(_call_latencies(task_rows)), + cost_usd=(sum(known_costs) / len(known_costs) if known_costs else None), + ) + + latencies = _call_latencies(executed) + return EconomicsMeasurement( + tasks=tasks, + total_input_tokens=total_input if saw_input else None, + total_result_tokens=sum(row.total_result_tokens for row in executed), + total_wall_time_s=sum(float(row.wall_time_s) for row in executed), + cost_usd=cost_total if priced else None, + vendor_cost_usd=vendor_total if saw_vendor else None, + cost_outcome=outcome, + priced_rows=priced, + unpriced_rows=unpriced, + unmeasured_rows=unmeasured, + med_call_latency_ms=median(latencies), + p95_call_latency_ms=percentile(latencies, 0.95), + ) + + +def economics_statement(measurement: EconomicsMeasurement) -> str: + """One block naming cost, volume and latency, with unknowns named as unknowns.""" + input_text = f"{measurement.total_input_tokens:,}" if measurement.total_input_tokens is not None else UNMEASURED + latency = measurement.med_call_latency_ms + p95 = measurement.p95_call_latency_ms + latency_text = f"{latency:,.0f}ms median / {p95:,.0f}ms p95" if latency is not None and p95 is not None else "n/a" + lines = [ + f"economics: cost={measurement.cost_text} (prices as of {measurement.prices_as_of}); " + f"input tokens={input_text}; result tokens={measurement.total_result_tokens:,}", + f" wall time={measurement.total_wall_time_s:,.0f}s; call latency {latency_text}", + ] + if measurement.vendor_cost_usd is not None and measurement.cost_usd is not None: + # The only standing check that the price table has not gone stale. + drift = measurement.cost_usd - measurement.vendor_cost_usd + lines.append(f" vendor-reported cost=${measurement.vendor_cost_usd:,.3f}; table differs by ${drift:+,.3f}") + lines.append(f" {COST_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "COST_LIMITATION", + "EconomicsMeasurement", + "TaskEconomics", + "economics_statement", + "measure_economics", +] diff --git a/evals/report/summary.py b/evals/report/summary.py index e95b09a7..86f340a0 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -10,6 +10,7 @@ from evals.core.task_metadata import TaskMetadata, entry_needs, task_metadata_from_rows from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family +from .economics import EconomicsMeasurement, measure_economics from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import OffSurfaceMeasurement, measure_off_surface from .schema_friction import SchemaFrictionMeasurement, measure_schema_friction @@ -89,6 +90,7 @@ class Summary: result_tokens_mode: ResultTokensMode off_surface: OffSurfaceMeasurement schema_friction: SchemaFrictionMeasurement + economics: EconomicsMeasurement @property def complete(self) -> bool: @@ -385,4 +387,5 @@ def summarize( result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), off_surface=measure_off_surface(rows, task_catalog=task_metadata or None), schema_friction=measure_schema_friction(rows), + economics=measure_economics(rows), ) diff --git a/tests/evals/report/test_economics.py b/tests/evals/report/test_economics.py new file mode 100644 index 00000000..d026a68f --- /dev/null +++ b/tests/evals/report/test_economics.py @@ -0,0 +1,93 @@ +"""Cost, input volume, result volume and latency must reach the A/B view. + +These are all recorded already. Result tokens even reach ``--table``. They were +absent from the two-file comparison, which is the view a two-arm question is asked +in -- so the arm that made 32% fewer calls while burning 3.1x the input tokens read +as the efficient one for two days. +""" + +from __future__ import annotations + +from evals.core.pricing import PRICED, UNMEASURED +from evals.report import summarize +from evals.report.compare import ab_compare, print_ab_report + +OPENAI_USAGE = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", + "cache_semantics": "inclusive", +} +CODEX_USAGE = { + "input_tokens": 249983, + "output_tokens": 682, + "cache_read_input_tokens": 224768, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 474751, + "source": "codex_token_count", +} + + +def row(task_id, *, usage, model, calls=2, wall=10.0, latency=500.0, result_tokens=900): + return { + "task_id": task_id, + "success": True, + "trace_integrity": True, + "model": model, + "usage_total": usage, + "wall_time_s": wall, + "num_calls": calls, + "calls": [{"tool": "workitem", "duration_ms": latency, "result_tokens": result_tokens} for _ in range(calls)], + } + + +def test_summary_carries_arm_cost_and_normalised_input(): + economics = summarize([row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna")]).economics + assert economics.cost_outcome == PRICED + assert economics.cost_usd is not None and economics.cost_usd > 0 + assert economics.total_input_tokens == 97813 + assert economics.total_wall_time_s == 10.0 + assert economics.med_call_latency_ms == 500.0 + + +def test_an_arm_with_no_usage_reports_unmeasured_not_zero(): + """antigravity records usage on no row at all; $0.00 would read as free.""" + economics = summarize([row("R1", usage=None, model="gemini-3.6-flash-low")]).economics + assert economics.cost_outcome == UNMEASURED + assert economics.cost_usd is None + assert economics.unmeasured_rows == 1 + assert economics.cost_text == UNMEASURED + + +def test_ab_block_reports_the_metrics_that_invert_the_call_verdict(capsys): + """B makes fewer calls and costs far more -- both facts must be visible together.""" + rows_a = [row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna", calls=7, result_tokens=900)] + rows_b = [row("R1", usage=CODEX_USAGE, model="gpt-5.6-luna", calls=4, result_tokens=2500)] + comparison = ab_compare(rows_a, rows_b) + + assert comparison["total_input_a"] == 97813 + assert comparison["total_input_b"] == 474751 + assert comparison["cost_a"] is not None and comparison["cost_b"] is not None + assert comparison["cost_b"] > comparison["cost_a"] + + print_ab_report(comparison, "A.jsonl", "B.jsonl") + out = capsys.readouterr().out + for expected in ("input tokens", "cost", "result tokens", "wall time", "call latency"): + assert expected in out, f"{expected!r} missing from the A/B block" + # The call delta says B is better; the cost delta must be right there beside it. + assert "median call delta" in out + + +def test_unmeasured_arm_prints_a_word_not_a_zero(capsys): + rows_a = [row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna")] + rows_b = [row("R1", usage=None, model="gemini-3.6-flash-low")] + print_ab_report(ab_compare(rows_a, rows_b), "A.jsonl", "B.jsonl") + out = capsys.readouterr().out + b_lines = [line for line in out.splitlines() if line.startswith(" B economics:")] + assert b_lines, "arm B has no economics line" + assert "cost=unmeasured" in b_lines[0] + assert "input tokens=unmeasured" in b_lines[0] + # The unknown must never be rendered as a figure of any size, zero included. + assert "$" not in b_lines[0] From 27e3d134b5d7f1a1650953677cf7b56b337e99fe Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:09:55 +0530 Subject: [PATCH 86/93] feat(evals): classify failures by kind, not just pass/fail W7 put a link on the wrong work item, I1 set urgent where high was asked, and S2 spent 43 calls producing nothing. Three different defects, reported identically as a failed verifier, separable only by reading notes by hand. The pattern table was built from the notes actually on disk -- a scan of 100 distinct verifier notes across every recorded run -- rather than from guesses, and that scan turned up a family the plan had not anticipated and that dominates everything else: answer_correct=true with provenance missing or the trace incomplete. The agent answered correctly and the run could not evidence it. That is a harness property, not an agent defect, and at 54.3% of all failed rows in the corpus, folding it into the others would misattribute most failures to the model. It gets its own kind, and abandoned/environment join it as non-defects. Measured coverage on the corpus: unproven 54.3%, wrong_value 17.1%, partial_write 9.3%, abandoned 8.5%, missing_write 4.7%, environment 4.7%, unclassified 1.6% -- and unclassified is counted and printed, so a zero in some kind never stands in for "the classifier did not recognise it". Two notes on what this deliberately does not do. Kinds come from note text only, so a write that landed on the wrong entity reports as partial or missing -- the right entity is empty either way, and telling those apart needs call arguments. And the plan assumed stop_reason would classify S2 as abandoned; it does not, as S2 stopped voluntarily on end_turn after 43 calls. W7/I1/S2 land in three distinct kinds as intended. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/failure_kind.py | 134 +++++++++++++++++++++++++++++ evals/report/compare.py | 3 + evals/report/failure_kinds.py | 93 ++++++++++++++++++++ evals/report/summary.py | 3 + evals/report/table.py | 4 + tests/evals/report/test_summary.py | 5 ++ tests/evals/test_failure_kind.py | 120 ++++++++++++++++++++++++++ 7 files changed, 362 insertions(+) create mode 100644 evals/core/failure_kind.py create mode 100644 evals/report/failure_kinds.py create mode 100644 tests/evals/test_failure_kind.py diff --git a/evals/core/failure_kind.py b/evals/core/failure_kind.py new file mode 100644 index 00000000..06ee8985 --- /dev/null +++ b/evals/core/failure_kind.py @@ -0,0 +1,134 @@ +"""What kind of wrong a failed task was. + +One failed verifier answers several unrelated questions at once. In one measured +battery, ``W7`` put a link on the wrong work item, ``I1`` set ``urgent`` where +``high`` was asked, and ``S2`` spent 43 calls and produced nothing -- three +different defects, reported identically, separable only by reading notes by hand. + + unproven the answer was right and the run could not evidence it. Not an + agent defect at all, and the largest single family in the recorded + corpus, so folding it into the others would misattribute most + failures to the model. + wrong_value a value was written or reported, and it differs from the one asked + for. + missing_write the thing was never created; the verifier found nothing. + partial_write a multi-part change half landed -- the shape that hides a wrong + target, since writing correctly to the wrong entity leaves the + right entity empty. + abandoned the run hit its iteration or token ceiling, so the note describes + an unfinished state rather than a defect. + environment a capability the environment does not have. Not a defect either. + +Same contract as ``error_class``: a narrow pattern table over text the verifiers +own, and ``unclassified`` is a first-class member that is counted and printed. A +zero in some kind must mean "none of these", never "the classifier did not +recognise it". + +Deliberately note-only. Whether a write went to the *wrong target* is not knowable +from a note that reports the right target as empty -- that needs call arguments, +which is a different measurement. +""" + +from __future__ import annotations + +UNPROVEN = "unproven" +WRONG_VALUE = "wrong_value" +MISSING_WRITE = "missing_write" +PARTIAL_WRITE = "partial_write" +ABANDONED = "abandoned" +ENVIRONMENT = "environment" +UNCLASSIFIED = "unclassified" + +FAILURE_KINDS = ( + UNPROVEN, + WRONG_VALUE, + MISSING_WRITE, + PARTIAL_WRITE, + ABANDONED, + ENVIRONMENT, + UNCLASSIFIED, +) + +#: Kinds that are properties of the run or the environment, not of the agent. +NON_DEFECT_KINDS = (UNPROVEN, ENVIRONMENT, ABANDONED) + +#: stop_reason values that mean the run was cut off rather than finished. +_CAPPED_STOP_REASONS = frozenset({"max_turns", "max_tokens", "max_iterations"}) + +#: The verifier states its verdict before its evidence, so these settle the note. +_ANSWER_CORRECT = "answer_correct=true" +_ANSWER_WRONG = "answer_correct=false" + +#: Wording for something the verifier looked for and did not find. +_ABSENT = ( + "not found", + "was not created", + "missing", + "have []", +) + +#: Wording for something it did find. Only meaningful next to an absence, where the +#: pair means a change landed in part. +_PRESENT = ( + " present", + "names ", +) + +#: A stated expectation, which implies a value was compared rather than absent. +_EXPECTATION = ("(want ", "want ") + + +def classify_failure( + note: str | None, + *, + stop_reason: str | None = None, + hit_max_iterations: bool = False, +) -> str: + """Return the kind of failure a verifier note describes. + + Structural signals win over the note: a run that hit its ceiling has an + unfinished state to report regardless of what the note says about it. + """ + if hit_max_iterations or (stop_reason or "").strip().lower() in _CAPPED_STOP_REASONS: + return ABANDONED + + text = (note or "").strip() + if not text: + return UNCLASSIFIED + lowered = text.lower() + + if lowered.startswith("env:"): + return ENVIRONMENT + + # The verifier's own verdict outranks the prose after it: a note can say the + # answer was right and still be a failure, because the evidence was missing. + if _ANSWER_CORRECT in lowered: + return UNPROVEN + if _ANSWER_WRONG in lowered: + return WRONG_VALUE + + absent = any(marker in lowered for marker in _ABSENT) + present = any(marker in lowered for marker in _PRESENT) + if absent and present: + return PARTIAL_WRITE + if absent: + # Checked before the expectation markers on purpose. "missing X ... (want 5)" + # is nothing written, not a wrong value. + return MISSING_WRITE + if any(marker in lowered for marker in _EXPECTATION): + return WRONG_VALUE + return UNCLASSIFIED + + +__all__ = [ + "ABANDONED", + "ENVIRONMENT", + "FAILURE_KINDS", + "MISSING_WRITE", + "NON_DEFECT_KINDS", + "PARTIAL_WRITE", + "UNCLASSIFIED", + "UNPROVEN", + "WRONG_VALUE", + "classify_failure", +] diff --git a/evals/report/compare.py b/evals/report/compare.py index 32a21618..aefe3836 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -6,6 +6,7 @@ from typing import Any from .economics import economics_statement +from .failure_kinds import failure_kind_statement from .load import ResultRow, RunKeyValidation from .off_surface import off_surface_statement from .schema_friction import measure_schema_friction, schema_friction_statement, successful_trace_rows @@ -275,6 +276,8 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N print(f" {label} {line}") for line in schema_friction_statement(summary.schema_friction).splitlines(): print(f" {label} {line}") + for line in failure_kind_statement(summary.failure_kinds).splitlines(): + print(f" {label} {line}") print(f" A {completeness_statement(comparison['summary_a'])}") print(f" B {completeness_statement(comparison['summary_b'])}") print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") diff --git a/evals/report/failure_kinds.py b/evals/report/failure_kinds.py new file mode 100644 index 00000000..44d2afe5 --- /dev/null +++ b/evals/report/failure_kinds.py @@ -0,0 +1,93 @@ +"""Group a run's failures by what kind of wrong they were.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, field + +from evals.core.failure_kind import FAILURE_KINDS, NON_DEFECT_KINDS, UNCLASSIFIED, classify_failure +from evals.core.results import TaskResult + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result + +FAILURE_KIND_LIMITATION = ( + "limitation: kinds are read from verifier note text, so they describe what the verifier " + "could see. A write that landed on the wrong entity reports as a missing or partial write, " + "because the right entity is empty either way -- distinguishing those needs call arguments" +) + + +@dataclass(frozen=True, slots=True) +class FailureKindMeasurement: + """Counts per kind, plus the task ids behind each.""" + + counts: dict[str, int] = field(default_factory=dict) + task_ids: dict[str, tuple[str, ...]] = field(default_factory=dict) + total: int = 0 + + @property + def defects(self) -> int: + """Failures attributable to the agent rather than the run or environment.""" + return sum(count for kind, count in self.counts.items() if kind not in NON_DEFECT_KINDS) + + @property + def non_defects(self) -> int: + return sum(self.counts.get(kind, 0) for kind in NON_DEFECT_KINDS) + + +def measure_failure_kinds(rows: list[ResultRow]) -> FailureKindMeasurement: + """Classify every failed row that carries a verifier note.""" + counts: dict[str, int] = dict.fromkeys(FAILURE_KINDS, 0) + tasks: dict[str, set[str]] = defaultdict(set) + total = 0 + for raw_row in rows: + row: TaskResult = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + continue + if row.success: + continue + kind = classify_failure( + row.verify_note, + stop_reason=row.stop_reason, + hit_max_iterations=row.hit_max_iterations, + ) + counts[kind] += 1 + tasks[kind].add(row.task_id) + total += 1 + return FailureKindMeasurement( + counts=counts, + task_ids={kind: tuple(sorted(ids)) for kind, ids in sorted(tasks.items())}, + total=total, + ) + + +def failure_kind_statement(measurement: FailureKindMeasurement) -> str: + """Name every kind that occurred, and say plainly which are not agent defects.""" + if not measurement.total: + return "failure kinds: no failed rows" + parts = [ + f"{kind}={measurement.counts[kind]}" + + (f" [{', '.join(measurement.task_ids[kind])}]" if kind in measurement.task_ids else "") + for kind in FAILURE_KINDS + if measurement.counts.get(kind) + ] + lines = [f"failure kinds ({measurement.total} failed rows): " + "; ".join(parts)] + if measurement.non_defects: + lines.append( + f" {measurement.non_defects} of {measurement.total} are not agent defects: a correct answer " + "the run could not evidence, an environment gap, or a capped run" + ) + unclassified = measurement.counts.get(UNCLASSIFIED, 0) + if unclassified: + # Never let a zero in some kind stand in for "the classifier did not recognise it". + lines.append(f" {unclassified} note(s) matched no pattern, so the split above is incomplete by that much") + lines.append(f" {FAILURE_KIND_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "FAILURE_KIND_LIMITATION", + "FailureKindMeasurement", + "failure_kind_statement", + "measure_failure_kinds", +] diff --git a/evals/report/summary.py b/evals/report/summary.py index 86f340a0..ece8a441 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -11,6 +11,7 @@ from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family from .economics import EconomicsMeasurement, measure_economics +from .failure_kinds import FailureKindMeasurement, measure_failure_kinds from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import OffSurfaceMeasurement, measure_off_surface from .schema_friction import SchemaFrictionMeasurement, measure_schema_friction @@ -91,6 +92,7 @@ class Summary: off_surface: OffSurfaceMeasurement schema_friction: SchemaFrictionMeasurement economics: EconomicsMeasurement + failure_kinds: FailureKindMeasurement @property def complete(self) -> bool: @@ -388,4 +390,5 @@ def summarize( off_surface=measure_off_surface(rows, task_catalog=task_metadata or None), schema_friction=measure_schema_friction(rows), economics=measure_economics(rows), + failure_kinds=measure_failure_kinds(rows), ) diff --git a/evals/report/table.py b/evals/report/table.py index 2f11c417..345b680c 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -9,6 +9,8 @@ from evals.core.results import TaskResult from evals.core.task_metadata import TaskMetadata, entry_prompt, task_metadata_from_rows +from .economics import economics_statement +from .failure_kinds import failure_kind_statement from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import off_surface_statement from .schema_friction import schema_friction_statement @@ -99,6 +101,8 @@ def print_table(summary: Summary, title: str) -> None: print(execution_coverage_statement(summary)) print(off_surface_statement(summary.off_surface)) print(schema_friction_statement(summary.schema_friction)) + print(failure_kind_statement(summary.failure_kinds)) + print(economics_statement(summary.economics)) print(completeness_statement(summary)) if summary.infra_errors: print(f"infra errors: {summary.infra_errors}") diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 24862167..54def10e 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -411,6 +411,11 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): " limitation: a first not_found is read as the answer to an existence question, since asking has no " "cheaper form; only a repeat on the same tool and action is counted as friction. A surface that " "misleads an agent into one wrong lookup is therefore not charged for it\n" + "failure kinds: no failed rows\n" + "economics: cost=unmeasured (prices as of 2026-08-24); input tokens=unmeasured; result tokens=0\n" + " wall time=0s; call latency n/a\n" + " limitation: cost is computed from a static price table; a model absent from it reports " + "unpriced, and a row whose driver recorded no usage at all reports unmeasured. Neither is $0\n" "RUN COMPLETE: 1/1 rows completed\n" "tool variability: —\n" "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " diff --git a/tests/evals/test_failure_kind.py b/tests/evals/test_failure_kind.py new file mode 100644 index 00000000..cfc8429c --- /dev/null +++ b/tests/evals/test_failure_kind.py @@ -0,0 +1,120 @@ +"""The observed notes are the case table. + +Every string here was emitted by a real verifier in a recorded run; the scan behind +this file covered 100 distinct notes across every result file on disk. +""" + +from __future__ import annotations + +import pytest + +from evals.core.failure_kind import ( + ABANDONED, + ENVIRONMENT, + FAILURE_KINDS, + MISSING_WRITE, + PARTIAL_WRITE, + UNCLASSIFIED, + UNPROVEN, + WRONG_VALUE, + classify_failure, +) + + +def test_the_three_defects_that_motivated_this_are_distinct(): + """W7, I1 and S2 were three different defects reported identically as 'failed'.""" + w7 = classify_failure("blocking relation present; link 'https://example.com/eval/runbook-w7' missing; have []") + i1 = classify_failure("work_item 0cf779b6 priority='urgent' (want high)") + s2 = classify_failure("estimate points missing fib subset; have []; item estimate_point=None (want 5)") + assert w7 == PARTIAL_WRITE + assert i1 == WRONG_VALUE + assert s2 == MISSING_WRITE + assert len({w7, i1, s2}) == 3 + + +def test_a_correct_answer_the_harness_could_not_prove_is_not_an_agent_defect(): + """The largest family in the corpus. Counting these as defects would be wrong. + + The agent answered correctly; the run could not evidence it. That is a harness + property, and lumping it with wrong answers would misattribute the biggest + single group of failures to the model. + """ + for note in ( + "answer_correct=true (final text reports exactly 1 seeded comments); provenance=trace incomplete " + "(source=proxy; proxy sidecar was not authoritative)", + "answer_correct=true (final text reports activity count 1 via contract); provenance=missing " + "(0 evidence-bearing of 4 successful Plane calls; 5 total)", + ): + assert classify_failure(note) == UNPROVEN + + +def test_a_wrong_answer_is_a_wrong_value_even_when_provenance_is_named(): + note = ( + "answer_correct=false (logged-minutes values=['90', '90']; want ['90']); " + "provenance=observed seeded-value response evidence (source=proxy)" + ) + assert classify_failure(note) == WRONG_VALUE + + +@pytest.mark.parametrize( + "note", + [ + "customer 'Acme Corp' not found", + "Severity property not found on Bug type", + "project estimate not found; requested Fibonacci scale was not created", + ], +) +def test_absent_entities_are_missing_writes(note): + assert classify_failure(note) == MISSING_WRITE + + +def test_a_half_landed_write_is_partial_not_missing(): + assert classify_failure("names 1.2.0; missing changelog content") == PARTIAL_WRITE + + +@pytest.mark.parametrize( + "note", + [ + "state='Backlog' (want exact 'Done')", + "Sprint 12 not closed: end_date='2026-08-20T23:59:00Z' (want end_date='2026-08-19' or archived_at)", + ], +) +def test_value_mismatches_are_wrong_value(note): + assert classify_failure(note) == WRONG_VALUE + + +def test_environment_skips_are_not_defects(): + assert classify_failure("env:no-activity-worker") == ENVIRONMENT + + +def test_running_out_of_iterations_beats_whatever_the_note_says(): + """A capped run's note describes the unfinished state, not why it stopped.""" + assert classify_failure("estimate points missing fib subset", hit_max_iterations=True) == ABANDONED + assert classify_failure("anything at all", stop_reason="max_tokens") == ABANDONED + + +def test_s2_is_not_abandoned_despite_giving_up(): + """S2 spent 43 calls and stopped voluntarily with end_turn. + + The plan assumed stop_reason would classify this directly. It does not -- the + run ended normally, so only the note carries the defect. + """ + assert ( + classify_failure( + "estimate points missing fib subset; have []; item estimate_point=None (want 5)", + stop_reason="end_turn", + hit_max_iterations=False, + ) + == MISSING_WRITE + ) + + +def test_an_unrecognised_note_is_unclassified_never_silently_bucketed(): + """A zero in some kind must mean zero, not 'the classifier did not recognise it'.""" + assert classify_failure("3 module completed items archived") == UNCLASSIFIED + assert classify_failure("") == UNCLASSIFIED + assert classify_failure(None) == UNCLASSIFIED + + +def test_unclassified_is_a_first_class_member(): + assert UNCLASSIFIED in FAILURE_KINDS From e610d155f667afeb351482ecd0ede6ed725ee1ea Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:12:24 +0530 Subject: [PATCH 87/93] feat(evals): warn when per-task verdicts are underpowered A run's aggregate and its per-task rows have very different power and the report prints both in the same table. At 2 reps a task that passed once is 1/2 UNSTABLE with a 95% interval of about [0.09, 0.91] -- compatible with almost any true rate -- while the paired aggregate across 35 tasks resolved a call difference at p=0.0018. So a per-task row is least readable exactly when the aggregate is most convincing, and that asymmetry was misread. The line reports the best-covered task rather than the typical one: if any task reaches 5 reps then "no per-task verdict is supported" is false, and a caveat that overstates its scope gets ignored along with the ones that do not. No new machinery -- --tasks already allows a focused high-rep subset, and at ~$0.50 an arm that is affordable. Co-Authored-By: Claude Opus 5 (1M context) --- evals/report/compare.py | 5 +++ evals/report/power.py | 45 ++++++++++++++++++++++ evals/report/table.py | 4 ++ tests/evals/report/test_power.py | 61 ++++++++++++++++++++++++++++++ tests/evals/report/test_summary.py | 3 ++ 5 files changed, 118 insertions(+) create mode 100644 evals/report/power.py create mode 100644 tests/evals/report/test_power.py diff --git a/evals/report/compare.py b/evals/report/compare.py index aefe3836..250d9ffd 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -9,6 +9,7 @@ from .failure_kinds import failure_kind_statement from .load import ResultRow, RunKeyValidation from .off_surface import off_surface_statement +from .power import power_statement from .schema_friction import measure_schema_friction, schema_friction_statement, successful_trace_rows from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue from .summary import completeness_statement, execution_coverage_statement, summarize @@ -278,6 +279,10 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N print(f" {label} {line}") for line in failure_kind_statement(summary.failure_kinds).splitlines(): print(f" {label} {line}") + for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): + power = power_statement(summary) + if power: + print(f" {label} {power}") print(f" A {completeness_statement(comparison['summary_a'])}") print(f" B {completeness_statement(comparison['summary_b'])}") print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") diff --git a/evals/report/power.py b/evals/report/power.py new file mode 100644 index 00000000..ab5fa9cd --- /dev/null +++ b/evals/report/power.py @@ -0,0 +1,45 @@ +"""Say plainly when the per-task numbers cannot carry a verdict. + +A run's aggregate and its per-task rows have very different power, and the report +prints both in the same table. At 2 repetitions a task that passed once is +``1/2 UNSTABLE`` with a 95% interval of roughly [0.09, 0.91] -- compatible with +almost any true success rate -- while the paired aggregate across 35 tasks resolved +a call difference at p=0.0018. Reading a per-task row as a finding is therefore +wrong in exactly the runs where the aggregate is most convincing. + +No new machinery: ``--tasks`` already allows a focused high-rep subset, and at +roughly $0.50 an arm that is affordable. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover - import cycle guard, summary imports this module + from .summary import Summary + +#: Repetitions per task below which a per-task pass rate is not worth reading. +UNDERPOWERED_REPS = 5 + + +def power_statement(summary: Summary) -> str | None: + """Return the guardrail line, or None when at least one task is well powered. + + The claim is deliberately about the best-covered task: if any task reaches the + threshold, "no per-task verdict is supported" would be false, and a caveat that + overstates its own scope gets ignored along with the ones that do not. + """ + counts = [task.n for task in summary.tasks.values() if task.n] + if not counts: + return None + best = max(counts) + if best >= UNDERPOWERED_REPS: + return None + return ( + f"POWER: {best} repetition(s) per task at most — per-task pass rates and UNSTABLE flags " + f"are not verdicts at this depth; read the aggregate and paired deltas instead. " + f"Use --tasks with --reps {UNDERPOWERED_REPS}+ for a per-task claim." + ) + + +__all__ = ["UNDERPOWERED_REPS", "power_statement"] diff --git a/evals/report/table.py b/evals/report/table.py index 345b680c..e654fb75 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -13,6 +13,7 @@ from .failure_kinds import failure_kind_statement from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result from .off_surface import off_surface_statement +from .power import power_statement from .schema_friction import schema_friction_statement from .statistics import wilson_interval from .summary import ( @@ -101,6 +102,9 @@ def print_table(summary: Summary, title: str) -> None: print(execution_coverage_statement(summary)) print(off_surface_statement(summary.off_surface)) print(schema_friction_statement(summary.schema_friction)) + power = power_statement(summary) + if power: + print(power) print(failure_kind_statement(summary.failure_kinds)) print(economics_statement(summary.economics)) print(completeness_statement(summary)) diff --git a/tests/evals/report/test_power.py b/tests/evals/report/test_power.py new file mode 100644 index 00000000..3ab21fa8 --- /dev/null +++ b/tests/evals/report/test_power.py @@ -0,0 +1,61 @@ +"""At low rep counts the per-task verdicts support nothing; the report must say so. + +At 2 reps every mixed task is `1/2 UNSTABLE` with a 95% interval of [0.09, 0.91], +which is compatible with almost any true rate -- while the paired aggregate over 35 +tasks is well powered. That asymmetry is easy to misread, and was misread. +""" + +from __future__ import annotations + +from evals.report import summarize +from evals.report.power import UNDERPOWERED_REPS, power_statement + + +def rows_at(reps: int, tasks: int = 3) -> list[dict]: + return [ + { + "task_id": f"T{task}", + "rep": rep, + "success": rep % 2 == 0, + "trace_integrity": True, + "num_calls": 1, + "calls": [], + } + for task in range(tasks) + for rep in range(reps) + ] + + +def test_the_guardrail_appears_at_two_reps(): + statement = power_statement(summarize(rows_at(2))) + assert statement is not None + assert "per-task" in statement + assert "aggregate" in statement + + +def test_the_guardrail_is_absent_at_five_reps(): + assert power_statement(summarize(rows_at(UNDERPOWERED_REPS))) is None + + +def test_a_single_well_powered_task_suppresses_the_blanket_claim(): + """The line says no task is well powered, so one that is makes it false.""" + rows = rows_at(2, tasks=2) + [ + {"task_id": "T9", "rep": rep, "success": True, "trace_integrity": True, "num_calls": 1, "calls": []} + for rep in range(UNDERPOWERED_REPS) + ] + assert power_statement(summarize(rows)) is None + + +def test_the_guardrail_names_the_rep_count_it_saw(): + assert "2" in (power_statement(summarize(rows_at(2))) or "") + + +def test_no_evaluated_rows_produces_no_claim(): + assert power_statement(summarize([])) is None + + +def test_it_reaches_the_printed_report(capsys): + import evals.report as report_mod + + report_mod.print_table(summarize(rows_at(2)), "Summary: low-power.jsonl") + assert "per-task" in capsys.readouterr().out diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index 54def10e..c9899a8f 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -411,6 +411,9 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): " limitation: a first not_found is read as the answer to an existence question, since asking has no " "cheaper form; only a repeat on the same tool and action is counted as friction. A surface that " "misleads an agent into one wrong lookup is therefore not charged for it\n" + "POWER: 1 repetition(s) per task at most \u2014 per-task pass rates and UNSTABLE flags " + "are not verdicts at this depth; read the aggregate and paired deltas instead. " + "Use --tasks with --reps 5+ for a per-task claim.\n" "failure kinds: no failed rows\n" "economics: cost=unmeasured (prices as of 2026-08-24); input tokens=unmeasured; result tokens=0\n" " wall time=0s; call latency n/a\n" From 979b0cd778bdcea5bcdd9c10e0e824987a409f93 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:17:42 +0530 Subject: [PATCH 88/93] fix(evals): eight defects from the adversarial review of the pricing change An independent read of the two pricing commits found eight defects, and the three most serious all had the same shape the code was written to prevent: something unknown or unverified presenting as a confident number. All eight reproduced before being fixed, and each has a regression test. - Usage present but carrying no counts priced at $0.00. The api driver writes a usage_total on every run whether or not any turn reported usage, so a dict with a source and no tokens was routine -- and read as free. Such a shape is now unmeasured. - Rows whose verifier crashed, timed out or skipped after the model ran were dropped entirely, so real spend vanished from the totals and from the unmeasured counters alike. A row that carries usage is now counted however it ended. - The drift check compared the vendor against itself. cost_total accumulated billed_usd, which already prefers the vendor figure, so the one mechanism the design leaned on to catch a stale price table always reported $0.000 and was structurally incapable of firing. The table's own figure is now kept separate. The unit test passed throughout because it called price_usage directly and never went through the report. - Partial coverage rendered as representative totals: input totals carried no missing-row annotation and paired deltas silently dropped tasks. Both now say what they left out, since missingness correlated with expensive tasks can reverse a delta. - A declared cache semantics suppressed validation against an explicit total, so a contradiction was accepted rather than refused. - Multi-model modelUsage fell through to the row alias, pricing Opus tokens at whatever rate that alias resolved to. Now unpriced. - An authoritative vendor cost was discarded when the table could not price the model. - Sub-cent costs rendered as $0.000, and an undeclared backend was recorded as explicitly exclusive rather than left to inference. Acceptance numbers unchanged: api $0.502, codex-cli $3.238, agy unmeasured, and the claude arm's computed cost still matches its reported $3.181 -- now as a real comparison rather than a tautology. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/pricing.py | 13 +- evals/core/token_accounting.py | 34 ++++- evals/drivers/api/driver.py | 19 ++- evals/report/compare.py | 9 +- evals/report/economics.py | 102 +++++++++++---- tests/evals/drivers/test_api_driver.py | 4 +- tests/evals/report/test_economics_review.py | 132 ++++++++++++++++++++ 7 files changed, 276 insertions(+), 37 deletions(-) create mode 100644 tests/evals/report/test_economics_review.py diff --git a/evals/core/pricing.py b/evals/core/pricing.py index d557483c..52618638 100644 --- a/evals/core/pricing.py +++ b/evals/core/pricing.py @@ -26,7 +26,7 @@ from dataclasses import dataclass from typing import Any -from evals.core.token_accounting import normalize_usage +from evals.core.token_accounting import has_token_counts, normalize_usage #: The day the rates below were last checked against published pricing. PRICES_AS_OF = "2026-08-24" @@ -95,7 +95,13 @@ def resolve_model_id(usage_total: Mapping[str, Any] | None, *, model: str | None """ if usage_total: model_usage = usage_total.get("modelUsage") - if isinstance(model_usage, Mapping) and len(model_usage) == 1: + if isinstance(model_usage, Mapping) and model_usage: + if len(model_usage) > 1: + # Several models produced this run and the counters are already summed, + # so no single rate is correct for them. Falling back to the row alias + # would price Opus tokens at the Haiku rate whenever that alias happened + # to be priceable. + return None only = next(iter(model_usage)) if isinstance(only, str) and only: return only @@ -114,7 +120,8 @@ def lookup_price(model_id: str | None) -> ModelPrice | None: def price_usage(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> RowCost: """Price one row's ``usage_total``.""" - if not usage_total: + if not usage_total or not has_token_counts(usage_total): + # A usage dict with no counts in it is metadata, not a measurement. return RowCost(outcome=UNMEASURED, usd=None, model_id=model or None) vendor = usage_total.get("total_cost_usd") diff --git a/evals/core/token_accounting.py b/evals/core/token_accounting.py index 3f687051..f3cad08f 100644 --- a/evals/core/token_accounting.py +++ b/evals/core/token_accounting.py @@ -76,6 +76,29 @@ def _family_semantics(model: str | None) -> str | None: return None +#: The fields that carry an actual measurement, as opposed to describing one. +_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "total_input_tokens_including_cache", +) + + +def has_token_counts(usage_total: Mapping[str, Any] | None) -> bool: + """True when this usage carries a real measurement rather than only metadata. + + The api driver writes a ``usage_total`` on every run whether or not any turn + reported usage, so a dict holding a ``source`` and a ``cache_semantics`` and no + counts is routine. Pricing that at $0.00 is the "unknown reads as free" failure + this module exists to prevent, so an all-zero shape counts as no measurement. + """ + if not usage_total: + return False + return any(_int(usage_total, name) for name in _TOKEN_FIELDS) + + def cache_semantics_of(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> str | None: """Return how this row's ``input_tokens`` treats cache, or None if undecidable.""" if not usage_total: @@ -102,7 +125,7 @@ def normalize_usage( means the driver recorded no usage at all (report it as unmeasured), while a populated one means the shape could not be read (report it as unpriced). """ - if not usage_total: + if not usage_total or not has_token_counts(usage_total): return None cached = _int(usage_total, "cache_read_input_tokens") @@ -132,12 +155,15 @@ def normalize_usage( total = reported_input uncached = total - cached - creation - if source == "explicit_total": + if explicit_total is not None and total != _int(usage_total, "total_input_tokens_including_cache"): # The vendor states the total as well as the parts. Both agreeing is what makes # this shape self-validating; disagreement means the shape changed underneath us, # and an unpriced row is a visible failure where a wrong price is not. - if total != _int(usage_total, "total_input_tokens_including_cache"): - return None + # + # Checked whenever a total is present, not only when it chose the semantics: a + # declaration may interpret the parts, but it does not get to overrule + # arithmetic that contradicts it. + return None if uncached < 0: # An inclusive reading whose cache exceeds its total is not a reading at all. diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py index 592d0c04..a4dfec10 100644 --- a/evals/drivers/api/driver.py +++ b/evals/drivers/api/driver.py @@ -48,6 +48,21 @@ DEFAULT_MAX_TOKENS = 8192 + +def cache_semantics_for(backend: Any) -> str | None: + """Return how this backend's input_tokens treats cache, or None if it never said. + + Defaulting an undeclared backend to either answer is a guess, and a guess written + down as a declaration is worse than no record: it outranks every other signal in + token_accounting. A registered extension that omits the attribute therefore + records nothing, and inference resolves it downstream -- or refuses. + """ + declared = getattr(backend, "input_tokens_include_cache", None) + if declared is None: + return None + return INCLUSIVE if declared else EXCLUSIVE + + McpSessionFactory = Callable[[StdioServerParameters], Any] @@ -420,8 +435,10 @@ async def _run_task( # the cached reads, and both arrive here as source "iterations". Recording # which one this was is the difference between pricing a run and guessing # at it from the model name. - "cache_semantics": (INCLUSIVE if getattr(backend, "input_tokens_include_cache", False) else EXCLUSIVE), } + semantics = cache_semantics_for(backend) + if semantics is not None: + usage_total["cache_semantics"] = semantics if manifest_state["stale"]: tool_manifest_fingerprint = None return AgentRun( diff --git a/evals/report/compare.py b/evals/report/compare.py index 250d9ffd..40a27451 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -40,6 +40,9 @@ def _paired_metric( deltas.append(float(value_b) - float(value_a)) return { "n": len(deltas), + # Missingness that correlates with expensive tasks can bias a delta, so the + # count of shared tasks this metric could not use travels with it. + "dropped": len(shared) - len(deltas), "mean_delta": sum(deltas) / len(deltas) if deltas else None, "median_delta": median(deltas), "ci": paired_bootstrap_mean_ci(deltas), @@ -212,7 +215,7 @@ def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: _RESOURCE_LINES: tuple[tuple[str, str, str, int], ...] = ( ("input", "input tokens", "", 0), ("result_tokens", "result tokens", "", 0), - ("cost", "cost", "$", 4), + ("cost", "cost per successful rep", "$", 4), ("wall_time", "wall time", "s", 1), ("call_latency", "call latency", "ms", 0), ) @@ -231,6 +234,8 @@ def _print_resource_deltas(comparison: dict[str, Any]) -> None: if not metric or not metric["n"]: print(f" median {label} delta (B−A): n/a (no paired tasks reporting it)") continue + dropped = metric.get("dropped") or 0 + omitted = f", {dropped} shared task(s) omitted for missing values" if dropped else "" low, high = metric["ci"] prefix = unit if unit == "$" else "" suffix = "" if unit == "$" else unit @@ -241,7 +246,7 @@ def _print_resource_deltas(comparison: dict[str, Any]) -> None: ) print( f" median {label} delta (B−A): {prefix}{metric['median_delta']:+,.{places}f}{suffix}" - f"{interval} (n={metric['n']} tasks)" + f"{interval} (n={metric['n']} tasks{omitted})" ) for label, key in (("A", "economics_a"), ("B", "economics_b")): economics = comparison.get(key) diff --git a/evals/report/economics.py b/evals/report/economics.py index b281fb65..4fc68950 100644 --- a/evals/report/economics.py +++ b/evals/report/economics.py @@ -20,7 +20,7 @@ from evals.core.pricing import PRICED, PRICES_AS_OF, UNMEASURED, UNPRICED, price_usage from evals.core.results import TaskResult -from evals.core.token_accounting import normalize_usage +from evals.core.token_accounting import has_token_counts, normalize_usage from .load import ResultRow, is_infra_error_row, is_meta_row, read_result from .schema_friction import successful_trace_rows @@ -43,6 +43,7 @@ class TaskEconomics: med_wall_time_s: float | None med_call_latency_ms: float | None cost_usd: float | None + """Mean billed cost per successful repetition -- not a total, unlike the arm figure.""" @dataclass(frozen=True, slots=True) @@ -54,34 +55,67 @@ class EconomicsMeasurement: total_result_tokens: int total_wall_time_s: float cost_usd: float | None + computed_cost_usd: float | None vendor_cost_usd: float | None cost_outcome: str priced_rows: int unpriced_rows: int unmeasured_rows: int + missing_input_rows: int med_call_latency_ms: float | None p95_call_latency_ms: float | None prices_as_of: str = PRICES_AS_OF + @property + def cost_drift_usd(self) -> float | None: + """How far the price table sits from what the vendor said it charged. + + None when no vendor reported a figure, which is most runs. + """ + if self.computed_cost_usd is None or self.vendor_cost_usd is None: + return None + return self.computed_cost_usd - self.vendor_cost_usd + @property def cost_text(self) -> str: - """Never render an unknown cost as a number.""" - if self.cost_outcome == UNMEASURED or self.cost_usd is None: + """Never render an unknown cost as a number, or a real one as zero.""" + if self.cost_usd is None: return UNMEASURED if self.cost_outcome == UNMEASURED else UNPRICED - text = f"${self.cost_usd:,.3f}" + # A run that really cost a fraction of a cent must not print $0.000; that is + # the same "reads as free" mistake in a different disguise. + text = "<$0.001" if 0 < self.cost_usd < 0.001 else f"${self.cost_usd:,.3f}" if self.unpriced_rows or self.unmeasured_rows: text += f" (+{self.unpriced_rows} unpriced, {self.unmeasured_rows} unmeasured rows)" return text + @property + def input_text(self) -> str: + """The input total, saying so when it does not cover every row.""" + if self.total_input_tokens is None: + return UNMEASURED + text = f"{self.total_input_tokens:,}" + if self.missing_input_rows: + text += f" (excludes {self.missing_input_rows} row(s) with unreadable usage)" + return text + + +def _charged_rows(rows: list[ResultRow]) -> list[TaskResult]: + """Every row whose model actually ran, including ones that later went wrong. -def _executed_rows(rows: list[ResultRow]) -> list[TaskResult]: - executed: list[TaskResult] = [] + A verifier crash, a contained timeout or a post-run skip happens after the tokens + were spent, so excluding those rows understates an arm and hides the spend + entirely -- it was not even counted as unmeasured. A row that carries usage is + kept regardless of how it ended; one that never ran is not. + """ + charged: list[TaskResult] = [] for raw_row in rows: row = read_result(raw_row) - if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + if is_meta_row(row) or is_infra_error_row(row): + continue + if (row.error or row.skipped) and not has_token_counts(row.usage_total): continue - executed.append(row) - return executed + charged.append(row) + return charged def _row_input_tokens(row: TaskResult) -> int | None: @@ -95,11 +129,15 @@ def _call_latencies(rows: list[TaskResult]) -> list[float]: def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: """Total an arm's cost and volume, and break it down per task.""" - executed = _executed_rows(rows) + executed = _charged_rows(rows) total_input = 0 saw_input = False - cost_total = 0.0 + missing_input = 0 + billed_total = 0.0 + saw_billed = False + computed_total = 0.0 + saw_computed = False vendor_total = 0.0 saw_vendor = False priced = unpriced = unmeasured = 0 @@ -108,26 +146,34 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: if tokens is not None: total_input += tokens saw_input = True + else: + missing_input += 1 cost = price_usage(row.usage_total, model=row.model) if cost.outcome == PRICED: priced += 1 - if cost.billed_usd is not None: - cost_total += cost.billed_usd elif cost.outcome == UNPRICED: unpriced += 1 else: unmeasured += 1 + # Billed is what to report; computed is the table's own opinion, kept apart so + # the drift check compares the table against the vendor rather than the vendor + # against itself. Billed accrues on any row that has a figure, so an + # authoritative vendor cost survives a model the table cannot price. + if cost.billed_usd is not None: + billed_total += cost.billed_usd + saw_billed = True + if cost.usd is not None: + computed_total += cost.usd + saw_computed = True if cost.vendor_usd is not None: vendor_total += cost.vendor_usd saw_vendor = True - if priced == 0: - # No priced row at all: say which kind of nothing this is. + if not saw_billed: + # Nothing to report: say which kind of nothing it is. outcome = UNMEASURED if unpriced == 0 else UNPRICED - elif unpriced or unmeasured: - outcome = PRICED # partial, and the counts are carried alongside else: - outcome = PRICED + outcome = PRICED # possibly partial; the counts travel alongside by_task: dict[str, list[TaskResult]] = defaultdict(list) for row in successful_trace_rows(rows): @@ -155,8 +201,10 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: total_input_tokens=total_input if saw_input else None, total_result_tokens=sum(row.total_result_tokens for row in executed), total_wall_time_s=sum(float(row.wall_time_s) for row in executed), - cost_usd=cost_total if priced else None, + cost_usd=billed_total if saw_billed else None, + computed_cost_usd=computed_total if saw_computed else None, vendor_cost_usd=vendor_total if saw_vendor else None, + missing_input_rows=missing_input, cost_outcome=outcome, priced_rows=priced, unpriced_rows=unpriced, @@ -168,7 +216,7 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: def economics_statement(measurement: EconomicsMeasurement) -> str: """One block naming cost, volume and latency, with unknowns named as unknowns.""" - input_text = f"{measurement.total_input_tokens:,}" if measurement.total_input_tokens is not None else UNMEASURED + input_text = measurement.input_text latency = measurement.med_call_latency_ms p95 = measurement.p95_call_latency_ms latency_text = f"{latency:,.0f}ms median / {p95:,.0f}ms p95" if latency is not None and p95 is not None else "n/a" @@ -177,10 +225,16 @@ def economics_statement(measurement: EconomicsMeasurement) -> str: f"input tokens={input_text}; result tokens={measurement.total_result_tokens:,}", f" wall time={measurement.total_wall_time_s:,.0f}s; call latency {latency_text}", ] - if measurement.vendor_cost_usd is not None and measurement.cost_usd is not None: - # The only standing check that the price table has not gone stale. - drift = measurement.cost_usd - measurement.vendor_cost_usd - lines.append(f" vendor-reported cost=${measurement.vendor_cost_usd:,.3f}; table differs by ${drift:+,.3f}") + drift = measurement.cost_drift_usd + if drift is not None and measurement.vendor_cost_usd is not None: + # The only standing check that the price table has not gone stale, so it has to + # compare the table's own figure against the vendor's -- not the reported cost, + # which already prefers the vendor and would always agree with itself. + lines.append( + f" vendor-reported cost=${measurement.vendor_cost_usd:,.3f}; " + f"price table computes ${measurement.computed_cost_usd:,.3f} " + f"(differs by ${drift:+,.3f})" + ) lines.append(f" {COST_LIMITATION}") return "\n".join(lines) diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py index 15b183af..0fa7890f 100644 --- a/tests/evals/drivers/test_api_driver.py +++ b/tests/evals/drivers/test_api_driver.py @@ -838,9 +838,7 @@ def _openai_backend_preserves_malformed_arguments(): { "model": "gpt", "status": "completed", - "output": [ - {"type": "function_call", "call_id": "c1", "name": "lookup", "arguments": "{not json"} - ], + "output": [{"type": "function_call", "call_id": "c1", "name": "lookup", "arguments": "{not json"}], "usage": None, } ] diff --git a/tests/evals/report/test_economics_review.py b/tests/evals/report/test_economics_review.py new file mode 100644 index 00000000..eca524d6 --- /dev/null +++ b/tests/evals/report/test_economics_review.py @@ -0,0 +1,132 @@ +"""Regressions from the adversarial review of the pricing and economics change. + +Eight findings, all reproduced before being fixed. The three most serious all had +the same shape: something unknown or unverified presenting as a confident number, +which is the exact failure this code was written to prevent. +""" + +from __future__ import annotations + +from evals.core.pricing import PRICED, UNMEASURED, UNPRICED, price_usage, resolve_model_id +from evals.core.token_accounting import EXCLUSIVE, INCLUSIVE, normalize_usage +from evals.report.economics import measure_economics + +CLAUDE_USAGE = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 9.0, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + "source": "modelUsage", +} + + +def row(**overrides): + base = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "model": "haiku", + "num_calls": 1, + "calls": [], + "usage_total": dict(CLAUDE_USAGE), + } + base.update(overrides) + return base + + +def test_f1_usage_present_but_carrying_no_tokens_is_unmeasured(): + """The api driver always writes a usage_total, even when every turn returned None. + + A dict with a source and no counts is not a measurement, and pricing it at + $0.00 is the precise "unknown reads as free" bug being designed against. + """ + empty = {"source": "iterations", "cache_semantics": INCLUSIVE} + assert normalize_usage(empty, model="gpt-5.6-luna") is None + assert price_usage(empty, model="gpt-5.6-luna").outcome == UNMEASURED + + +def test_f2_a_charged_row_whose_verifier_crashed_still_counts(): + """The model ran and the tokens were billed; a later crash does not refund them.""" + measurement = measure_economics([row(error="verifier crashed")]) + assert measurement.priced_rows == 1 + assert measurement.cost_usd is not None + + +def test_f3_drift_compares_the_table_against_the_vendor_not_the_vendor_against_itself(): + """The staleness detector was summing billed_usd, which already prefers vendor cost. + + Reported drift was therefore always $0.000 -- the one check the design leaned on + was structurally incapable of firing. + """ + measurement = measure_economics([row()]) + assert measurement.vendor_cost_usd == 9.0 + assert measurement.computed_cost_usd is not None + # The real table price for this row is cents, so drift against a $9 vendor + # figure must be large and negative. + assert measurement.computed_cost_usd < 1.0 + assert measurement.cost_drift_usd is not None and measurement.cost_drift_usd < -8.0 + + +def test_f4_a_partial_input_total_says_it_is_partial(): + priced = row() + blind = row(task_id="R2", usage_total={"input_tokens": 5, "cache_read_input_tokens": 4}, model="mystery") + measurement = measure_economics([priced, blind]) + assert measurement.missing_input_rows == 1 + assert "1 row" in measurement.input_text + + +def test_f5_a_declaration_that_contradicts_an_explicit_total_is_refused(): + """Declared semantics may interpret, but they may not override arithmetic.""" + contradictory = { + "input_tokens": 100, + "cache_read_input_tokens": 90, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 190, + "cache_semantics": INCLUSIVE, + } + assert normalize_usage(contradictory) is None + # Agreement still resolves normally. + consistent = dict(contradictory, cache_semantics=EXCLUSIVE) + accounting = normalize_usage(consistent) + assert accounting is not None and accounting.total_input == 190 + + +def test_f6_multi_model_usage_is_unpriced_rather_than_billed_at_one_rate(): + """Haiku and Opus tokens summed and priced at whichever alias the row carried.""" + usage = dict(CLAUDE_USAGE, modelUsage={"claude-haiku-4-5-20251001": {}, "claude-opus-5": {}}) + assert resolve_model_id(usage, model="claude-haiku-4-5") is None + assert price_usage(usage, model="claude-haiku-4-5").outcome == UNPRICED + + +def test_f6b_an_authoritative_vendor_cost_survives_an_unpriced_table_lookup(): + """A real billed figure must not vanish because the table lacks the model.""" + usage = dict(CLAUDE_USAGE, modelUsage={"a": {}, "b": {}}) + measurement = measure_economics([row(usage_total=usage, model="mystery")]) + assert measurement.unpriced_rows == 1 + assert measurement.vendor_cost_usd == 9.0 + assert measurement.cost_usd == 9.0 + + +def test_f7_a_small_real_cost_does_not_render_as_zero(): + tiny = {"input_tokens": 100, "output_tokens": 1, "cache_read_input_tokens": 0, "source": "iterations"} + measurement = measure_economics([row(usage_total=tiny, model="gpt-5.6-luna")]) + assert measurement.cost_outcome == PRICED + assert "$0.000 " not in measurement.cost_text + assert measurement.cost_text.startswith("$") or measurement.cost_text.startswith("<$") + + +def test_f8_a_backend_that_declares_nothing_is_not_recorded_as_exclusive(): + """getattr(..., False) turned an undeclared backend into a confident claim.""" + from evals.drivers.api.driver import cache_semantics_for + + class Undeclared: + pass + + class Inclusive: + input_tokens_include_cache = True + + assert cache_semantics_for(Undeclared()) is None + assert cache_semantics_for(Inclusive()) == INCLUSIVE From 7352003fb9f0247c77cd26de14f87472ff2ccf93 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:22:08 +0530 Subject: [PATCH 89/93] feat(evals): measure redundant lookups, and record call args on every driver The sharpest finding of the cross-harness pair was workitem.search 111 vs 26 against workitem.retrieve 4 vs 20: one arm carried resolved ids across turns and the other went looking again. That is a property of the surface as much as of the agent -- stickier identifiers would close it without either agent changing -- and nothing measured it. Measuring it needed call arguments, which the api driver recorded on none of its 484 calls while a CLI arm recorded them on 330 of 331. The cause was a coupling rather than a policy: args_json was conditioned on result_text, which only the recording proxy sets, and the api driver calls tools directly. The arguments were in hand on both paths the whole time -- args_chars is computed from them, and `action` was already persisted unconditionally -- so they are now recorded either way. What this adds to a result file is ids and short strings. That changes a deliberate earlier decision, so the test guarding it was rewritten to the new contract rather than deleted; it still guards the hop chain, which is where a field of this kind gets silently dropped. The detector is scoped to one row, since each repetition is a fresh conversation, and it never charges the lookup that first resolves an id. A run without recorded arguments reports "not measured" rather than zero -- "no redundant lookups" and "we could not tell" are opposite conclusions, and the pre-fix api arms are the latter. Measured: codex-cli 27 (19 on workitem), claude 8, and the old api arm correctly reports not measured. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/results.py | 14 ++- evals/report/compare.py | 1 + evals/report/lookup_reuse.py | 116 ++++++++++++++++++++++++ evals/report/summary.py | 3 + evals/report/table.py | 1 + tests/evals/report/test_lookup_reuse.py | 88 ++++++++++++++++++ tests/evals/report/test_summary.py | 1 + tests/evals/test_error_class.py | 19 ++-- 8 files changed, 232 insertions(+), 11 deletions(-) create mode 100644 evals/report/lookup_reuse.py create mode 100644 tests/evals/report/test_lookup_reuse.py diff --git a/evals/core/results.py b/evals/core/results.py index bc93204a..7e31b189 100644 --- a/evals/core/results.py +++ b/evals/core/results.py @@ -595,11 +595,15 @@ def agent_run_to_task_result( # tool choice — keep it (args content is otherwise not persisted). if isinstance(args, dict) and isinstance(args.get("action"), str): rec.action = args["action"] - # Under --record-result-payloads the proxy has already put the result body on the row, - # so the request that produced it is the other half of the same record. Keyed off - # result_text rather than a new flag: that is the signal payload recording is on, and - # a recorded response with no recorded request cannot be attributed to a target. - if isinstance(c.get("result_text"), str) and isinstance(args, dict) and args: + # Arguments are recorded for every driver. They were previously conditioned on + # result_text, which only the recording proxy sets -- so the api driver, which + # calls tools directly and never goes through the proxy, recorded arguments on + # none of its calls while a CLI arm recorded them on all of theirs. That is a + # coupling to an unrelated flag rather than a policy: the arguments are in hand + # on both paths, args_chars is already computed from them, and `action` above is + # already persisted unconditionally. What this adds to a result file is ids and + # short strings. + if isinstance(args, dict) and args: try: rec.args_json = json.dumps(args, default=str, ensure_ascii=False) except Exception: diff --git a/evals/report/compare.py b/evals/report/compare.py index 40a27451..da069d16 100644 --- a/evals/report/compare.py +++ b/evals/report/compare.py @@ -284,6 +284,7 @@ def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> N print(f" {label} {line}") for line in failure_kind_statement(summary.failure_kinds).splitlines(): print(f" {label} {line}") + print(f" {label} {summary.lookup_reuse.statement()}") for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): power = power_statement(summary) if power: diff --git a/evals/report/lookup_reuse.py b/evals/report/lookup_reuse.py new file mode 100644 index 00000000..02446b7c --- /dev/null +++ b/evals/report/lookup_reuse.py @@ -0,0 +1,116 @@ +"""How often an agent re-hunts an entity whose identifier it already holds. + +The sharpest single finding of the 2026-08-24 cross-harness pair was +``workitem.search`` 111 vs 26 against ``workitem.retrieve`` 4 vs 20. One arm carried +resolved ids across turns; the other went looking again each time. That is a +property of the **surface** as much as of the agent -- identifiers that stayed +sticky would close the gap without either agent changing -- and nothing measured it. + +Read from recorded call arguments, so it is only answerable on runs that have them. +A run that does not is reported as not measured, never as zero: "no redundant +lookups" and "we could not tell" are opposite conclusions. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass, field + +from evals.core.results import CallRecord, TaskResult + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result + +#: Actions that go looking for something rather than addressing it directly. +_SEARCH_ACTIONS = frozenset({"search", "list"}) + +#: Argument names that carry an identifier this run already resolved. Matched by +#: suffix so ``workitem_id``, ``parent_id`` and ``id`` all count. +_ID_SUFFIX = "_id" + + +def _args_of(call: CallRecord) -> dict | None: + if not call.args_json: + return None + try: + parsed = json.loads(call.args_json) + except (TypeError, ValueError): + return None + return parsed if isinstance(parsed, dict) else None + + +def _ids_in(args: dict) -> set[str]: + found: set[str] = set() + for name, value in args.items(): + if not isinstance(value, str) or not value: + continue + if name == "id" or name.endswith(_ID_SUFFIX): + found.add(value) + return found + + +@dataclass(frozen=True, slots=True) +class LookupReuseMeasurement: + """Redundant lookups, and whether the question could be asked at all.""" + + total: int = 0 + by_resource: dict[str, int] = field(default_factory=dict) + rows_measured: int = 0 + rows_without_args: int = 0 + + @property + def measurable(self) -> bool: + return self.rows_measured > 0 + + def statement(self) -> str: + if not self.measurable: + return f"redundant lookups: not measured — {self.rows_without_args} row(s) carry no recorded call arguments" + detail = ", ".join(f"{resource}={count}" for resource, count in sorted(self.by_resource.items())) + line = f"redundant lookups: {self.total} (a search or list on a resource whose id was already in hand)" + if detail: + line += f" [{detail}]" + if self.rows_without_args: + line += f"; {self.rows_without_args} row(s) not measured for want of arguments" + return line + + +def measure_lookup_reuse(rows: list[ResultRow]) -> LookupReuseMeasurement: + """Count searches issued after the same resource's id was already resolved. + + Scoped to one row. Each repetition is a fresh conversation, so an id learned in + one tells the agent in another nothing. + """ + total = 0 + by_resource: dict[str, int] = defaultdict(int) + measured = 0 + without_args = 0 + for raw_row in rows: + row: TaskResult = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + continue + if not any(call.args_json for call in row.calls): + without_args += 1 + continue + measured += 1 + known: dict[str, set[str]] = defaultdict(set) + for call in row.calls: + args = _args_of(call) + if args is None: + continue + resource = call.tool + action = (call.action or "").lower() + if action in _SEARCH_ACTIONS and known[resource]: + total += 1 + by_resource[resource] += 1 + # Learned after the check, so the call that first resolves an id is never + # charged for the lookup that produced it. + known[resource].update(_ids_in(args)) + return LookupReuseMeasurement( + total=total, + by_resource=dict(by_resource), + rows_measured=measured, + rows_without_args=without_args, + ) + + +__all__ = ["LookupReuseMeasurement", "measure_lookup_reuse"] diff --git a/evals/report/summary.py b/evals/report/summary.py index ece8a441..ce5e96bf 100644 --- a/evals/report/summary.py +++ b/evals/report/summary.py @@ -13,6 +13,7 @@ from .economics import EconomicsMeasurement, measure_economics from .failure_kinds import FailureKindMeasurement, measure_failure_kinds from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .lookup_reuse import LookupReuseMeasurement, measure_lookup_reuse from .off_surface import OffSurfaceMeasurement, measure_off_surface from .schema_friction import SchemaFrictionMeasurement, measure_schema_friction from .statistics import cluster_bootstrap_mean_ci, iqr, median, percentile, wilson_interval @@ -93,6 +94,7 @@ class Summary: schema_friction: SchemaFrictionMeasurement economics: EconomicsMeasurement failure_kinds: FailureKindMeasurement + lookup_reuse: LookupReuseMeasurement @property def complete(self) -> bool: @@ -391,4 +393,5 @@ def summarize( schema_friction=measure_schema_friction(rows), economics=measure_economics(rows), failure_kinds=measure_failure_kinds(rows), + lookup_reuse=measure_lookup_reuse(rows), ) diff --git a/evals/report/table.py b/evals/report/table.py index e654fb75..947a5ba9 100644 --- a/evals/report/table.py +++ b/evals/report/table.py @@ -106,6 +106,7 @@ def print_table(summary: Summary, title: str) -> None: if power: print(power) print(failure_kind_statement(summary.failure_kinds)) + print(summary.lookup_reuse.statement()) print(economics_statement(summary.economics)) print(completeness_statement(summary)) if summary.infra_errors: diff --git a/tests/evals/report/test_lookup_reuse.py b/tests/evals/report/test_lookup_reuse.py new file mode 100644 index 00000000..f2e6bc14 --- /dev/null +++ b/tests/evals/report/test_lookup_reuse.py @@ -0,0 +1,88 @@ +"""Re-hunting an entity whose id is already in hand is a surface property. + +The sharpest finding of the 2026-08-24 pair was workitem.search 111 vs 26 against +workitem.retrieve 4 vs 20: one arm carried resolved ids across turns and the other +re-searched for them. A surface with stickier identifiers would close that gap with +no change to either agent. +""" + +from __future__ import annotations + +from evals.report.lookup_reuse import measure_lookup_reuse + +WORKITEM_ID = "0cf779b6-9e18-4209-9e09-2264b889be42" + + +def call(tool, action, **args): + import json + + return {"tool": tool, "action": action, "args_json": json.dumps(args)} + + +def row(*calls, task_id="R1", rep=0): + return { + "task_id": task_id, + "rep": rep, + "success": True, + "trace_integrity": True, + "num_calls": len(calls), + "calls": list(calls), + } + + +def test_searching_for_something_already_in_hand_is_flagged(): + measurement = measure_lookup_reuse( + [row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), call("workitem", "search", query="thing"))] + ) + assert measurement.total == 1 + assert measurement.by_resource["workitem"] == 1 + + +def test_searching_before_any_id_is_known_is_not_flagged(): + """The first lookup is how the id is obtained; charging for it would be wrong.""" + measurement = measure_lookup_reuse( + [row(call("workitem", "search", query="thing"), call("workitem", "retrieve", workitem_id=WORKITEM_ID))] + ) + assert measurement.total == 0 + + +def test_a_search_on_a_different_resource_is_not_flagged(): + measurement = measure_lookup_reuse( + [row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), call("cycle", "list", project_id="p1"))] + ) + assert measurement.total == 0 + + +def test_ids_do_not_carry_across_rows(): + """Each repetition is a fresh conversation; nothing is in hand at its start.""" + first = row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), task_id="R1", rep=0) + second = row(call("workitem", "search", query="thing"), task_id="R1", rep=1) + assert measure_lookup_reuse([first, second]).total == 0 + + +def test_a_run_without_recorded_arguments_reports_that_it_cannot_tell(): + """Zero must never be reported when the question could not be asked.""" + blind = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [{"tool": "workitem", "action": "search"}, {"tool": "workitem", "action": "retrieve"}], + } + measurement = measure_lookup_reuse([blind]) + assert measurement.rows_without_args == 1 + assert measurement.measurable is False + assert "not measured" in measurement.statement() + + +def test_repeat_searches_after_the_id_is_known_each_count(): + measurement = measure_lookup_reuse( + [ + row( + call("workitem", "retrieve", workitem_id=WORKITEM_ID), + call("workitem", "search", query="a"), + call("workitem", "search", query="b"), + ) + ] + ) + assert measurement.total == 2 diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index c9899a8f..da740282 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -415,6 +415,7 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): "are not verdicts at this depth; read the aggregate and paired deltas instead. " "Use --tasks with --reps 5+ for a per-task claim.\n" "failure kinds: no failed rows\n" + "redundant lookups: not measured \u2014 1 row(s) carry no recorded call arguments\n" "economics: cost=unmeasured (prices as of 2026-08-24); input tokens=unmeasured; result tokens=0\n" " wall time=0s; call latency n/a\n" " limitation: cost is computed from a static price table; a model absent from it reports " diff --git a/tests/evals/test_error_class.py b/tests/evals/test_error_class.py index f6c9e627..3b9af4a9 100644 --- a/tests/evals/test_error_class.py +++ b/tests/evals/test_error_class.py @@ -224,14 +224,19 @@ def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): assert split_errors(reloaded.calls)["surface"] == 1, "the report did not see it" -def test_request_args_are_recorded_only_alongside_a_recorded_result(tmp_path): +def test_request_args_survive_the_hop_chain_on_every_driver(tmp_path): """A recorded refusal that cannot be attributed to a target answers half a question. W7 failed reproducibly with workitem_link.create reporting success while the link was absent, and the two candidate explanations -- wrong target, or a create that does not - persist -- were indistinguishable because only args_chars was kept. Args now ride along - with a recorded payload, and stay out when payloads are off: the same hop chain as - error_class, which is where a field of this kind gets silently dropped. + persist -- were indistinguishable because only args_chars was kept. + + Args used to ride along with a recorded payload and stay out otherwise. That coupling + was to result_text, which only the recording proxy sets, so the api driver -- which + calls tools directly -- recorded args on none of its 484 calls while a CLI arm recorded + them on 330 of 331. Since the arguments are in hand on both paths and `action` is + already kept unconditionally, they are now recorded either way. This still guards the + hop chain, which is where a field of this kind gets silently dropped. """ import json @@ -269,9 +274,11 @@ def roundtrip(*, with_payload: bool) -> TaskResult: # The target is the point: without it the record cannot say which item was linked. assert "wi-42" in recorded.calls[0].args_json - # Default stays metrics-only. args_chars is still there; the body is not. + # A call with no recorded payload keeps its args too -- that is the whole point of + # decoupling them, since the api driver never produces a payload to ride along with. plain = roundtrip(with_payload=False) - assert plain.calls[0].args_json is None, "args leaked into a run that did not ask for payloads" + assert plain.calls[0].args_json is not None, "args must not depend on payload recording" + assert json.loads(plain.calls[0].args_json) == args assert plain.calls[0].args_chars > 0 assert plain.calls[0].action == "create", "action is kept regardless — it is half the tool choice" From ed7c8a6cb577903b1ca3f3a97579a3fbfb5d4c37 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:31:31 +0530 Subject: [PATCH 90/93] docs(evals): describe cost, failure kinds, redundant lookups and power The args-recording paragraph had gone stale with the coupling change -- it still described arguments as riding along with --record-result-payloads. Adds the four new measurements, each stated with the distinction that motivates it: what an unknown cost is allowed to look like, that over half of failed rows are not agent defects, that a run without recorded arguments reports "not measured" rather than zero, and why a per-task row is least readable exactly when the aggregate is most convincing. Co-Authored-By: Claude Opus 5 (1M context) --- evals/DESIGN.md | 68 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/evals/DESIGN.md b/evals/DESIGN.md index 6fdb4fcb..1fbce88a 100644 --- a/evals/DESIGN.md +++ b/evals/DESIGN.md @@ -108,11 +108,17 @@ actually get told no about: a run measuring 12.8% refusals was really near 28%. deliberately narrow — only wording this server owns, and the stray-argument form must carry both halves of its sentence — so a result that merely quotes a refusal is not counted as one. -Recorded payloads carry the **request** as well (`args_json`, under -`--record-result-payloads`). Args are otherwise metrics-only: a recorded refusal that cannot be -attributed to the target it names answers half a question. This is what separated an agent -linking the wrong work item from a create that does not persist, when both fit the same -symptom. +Call **arguments** are recorded on every driver (`args_json`). A refusal that cannot be +attributed to the target it names answers half a question, and this is what separated an agent +linking the wrong work item from a create that does not persist, when both fit the same symptom. + +Arguments used to ride along with `--record-result-payloads` only. That coupled them to +`result_text`, which just the recording proxy sets, so the api driver — which calls tools +directly and never goes through the proxy — recorded arguments on none of its calls while a CLI +arm recorded them on nearly all of theirs. The arguments were in hand on both paths regardless: +`args_chars` is computed from them and `action` was already kept unconditionally. What a result +file gains is ids and short strings; what it gains in return is the redundant-lookup metric +below, which cannot be computed without them. Single-run success headlines use the same sampling unit: each evaluated task contributes its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. @@ -165,6 +171,58 @@ Payload recording is off by default because tool results contain live workspace make sidecars larger. The character-derived estimate remains useful for surface comparison because it is deterministic and monotonic in the recorded response size. +### Cost, and what an unknown cost is allowed to look like + +`usage_total.input_tokens` does not mean the same thing across drivers. It is **inclusive** of +cached reads under OpenAI Responses and **exclusive** under Anthropic Messages and every CLI +vendor, and the same api driver produces both — so driver family cannot decide it. Backends now +declare `input_tokens_include_cache` and the driver records `cache_semantics`; +`evals.core.token_accounting` resolves declared → explicit total → no cache activity → model +family, and **refuses** rather than guessing when none apply. Where a vendor states both parts +and a total, the two must agree or the row is not priced: that disagreement means the shape +changed underneath us, and an unpriced row is visible where a wrong price is not. + +Cost has three outcomes, and the distinction is the point. `priced`; `unpriced` when usage exists +but the model is not in the table; `unmeasured` when the driver recorded no usage at all, which is +true of every antigravity row. A silent `$0.00` reads as *free* rather than as *unknown*, so it is +never printed — nor is a real sub-cent cost rounded into one. + +Prices go stale and a date alone detects nothing. Claude Code reports `total_cost_usd` per run, so +the table's own figure is compared against the vendor's on every run that has one. That check must +compare the table against the vendor rather than the reported cost against itself — the reported +figure already prefers the vendor, so summing it on both sides makes the check incapable of firing. + +### Failure kinds + +A failed verifier answers several unrelated questions at once. Kinds are read from the verifier's +own note text: `unproven`, `wrong_value`, `missing_write`, `partial_write`, `abandoned`, +`environment`, `unclassified`. `unproven` — the answer was right and the run could not evidence it +— is the largest family in the recorded corpus at over half of all failed rows, and is not an agent +defect; nor are `environment` and `abandoned`. Reports name the non-defect total separately so a +raw failure count is not mistaken for a defect count. + +Kinds come from note text only, so a write that landed on the *wrong entity* reports as missing or +partial: the right entity is empty either way, and telling those apart needs call arguments. +`unclassified` is counted and printed, so a zero in some kind never stands in for "the classifier +did not recognise it". + +### Redundant lookups + +A `search` or `list` on a resource whose id already appeared in an earlier call of the same row. +This is a **surface** property as much as an agent one — identifiers that stayed sticky across +turns would close the gap without either agent changing. Scoped to one row, since each repetition +is a fresh conversation, and the call that first resolves an id is never charged for the lookup +that produced it. A run without recorded arguments reports *not measured* rather than zero: "no +redundant lookups" and "we could not tell" are opposite conclusions. + +### Power + +A run's aggregate and its per-task rows have very different power and appear in the same table. At +2 repetitions a task that passed once is `1/2 UNSTABLE` with a 95% interval of roughly [0.09, 0.91] +— compatible with almost any true rate — while a paired aggregate across 35 tasks can resolve a +difference at p=0.0018. Reports print a POWER line when no task reaches 5 repetitions, stating that +per-task verdicts are not supported at that depth. + ### Provenance: what counts as proof the answer came from the surface A read verifier asks two independent questions — is the answer right, and did the agent get From dec701e088bef62db2e10edf13ef212d4cd9adda Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:32:44 +0530 Subject: [PATCH 91/93] docs(evals): README no longer ties request args to payload recording Co-Authored-By: Claude Opus 5 (1M context) --- evals/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/evals/README.md b/evals/README.md index 90524276..0a9eb610 100644 --- a/evals/README.md +++ b/evals/README.md @@ -185,9 +185,10 @@ line (`N refusal(s) arrived flagged as successful results`) because it cannot jo on the protocol's error flag. It is worth watching: one measured surface refused roughly twice as often as its errored-call count implied. -`--record-result-payloads` keeps the request args beside each recorded result. Args are -metrics-only by default (`args_chars`), and a recorded result whose target is unknown cannot say -*which* item a call acted on — which is exactly the question a failing write raises. +Request args (`args_json`) are recorded on every driver, not just under +`--record-result-payloads`. A recorded result whose target is unknown cannot say *which* item a +call acted on — exactly the question a failing write raises — and the redundant-lookup metric +cannot be computed without them. Result *payloads* remain opt-in; args are ids and short strings. Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and From 3d7f0e8822b0bf27972be4ec4b763f9a0680c217 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 00:40:35 +0530 Subject: [PATCH 92/93] fix(evals): ten defects from the second adversarial review The two that mattered were a metric measuring something other than what it claimed, and a field documented as "ids and short strings" with no size bound. The redundant-lookup rule treated any *_id argument as the entity being in hand, so project_id alone triggered it. Pagination, a list after a create, and retrieving one item then searching for another were all counted. The rule now requires the resource's own identifier, ignores paginated continuations, and says in the report what it still cannot distinguish. Corrected numbers on the same arms: codex-cli 27 -> 2, claude 8 -> 0, clean-codex 25 -> 1. The earlier figures were almost entirely false positives, and the surface effect the metric was built to catch is not visible in arguments -- ids mostly arrive in results, which are not recorded by default. The metric undercounts by construction and now says so. args_json had no bound: a 1MB description_html was copied verbatim. The tool surface accepts rich HTML, page bodies, comment text and URLs, so "ids and short strings" was true of the current battery and of nothing else. Long values are truncated to 256 chars with structure preserved. Two comments still describing args as payload-gated were corrected. Also fixed: infrastructure-classified rows that had already burned tokens were dropped from cost entirely (the classification is applied after the run folds in); a vendor-only cost was discarded before being read; a capped run whose note proved a wrong answer was filed as abandoned, a non-defect; answer_correct=true was matched anywhere so a wrong value quoting it classified as unproven; "linked" was not a present marker and several real absence phrasings were unrecognised; the power line fell silent as soon as one task was deep, leaving shallow tasks uncaveated, and overstated what 5 reps settles; drift summed each side over different row populations, folding coverage differences into the comparison; and sub-cent figures still rendered as $0.000 in the drift line. Acceptance unchanged: api $0.502, codex-cli $3.238, agy unmeasured, claude table vs vendor agree over 70 rows carrying both. Co-Authored-By: Claude Opus 5 (1M context) --- evals/core/failure_kind.py | 33 +++- evals/core/pricing.py | 12 +- evals/core/results.py | 46 ++++- evals/report/economics.py | 71 ++++++-- evals/report/lookup_reuse.py | 83 ++++++--- evals/report/power.py | 21 ++- tests/evals/report/test_power.py | 14 +- tests/evals/report/test_review_round2.py | 222 +++++++++++++++++++++++ tests/evals/report/test_summary.py | 7 +- 9 files changed, 428 insertions(+), 81 deletions(-) create mode 100644 tests/evals/report/test_review_round2.py diff --git a/evals/core/failure_kind.py b/evals/core/failure_kind.py index 06ee8985..7cf4eb3e 100644 --- a/evals/core/failure_kind.py +++ b/evals/core/failure_kind.py @@ -31,6 +31,8 @@ from __future__ import annotations +import re + UNPROVEN = "unproven" WRONG_VALUE = "wrong_value" MISSING_WRITE = "missing_write" @@ -72,11 +74,17 @@ _PRESENT = ( " present", "names ", + " linked", ) #: A stated expectation, which implies a value was compared rather than absent. _EXPECTATION = ("(want ", "want ") +#: Absence phrasings that carry no "missing"/"not found" wording. Narrow on purpose -- +#: a bare "not " would swallow "not closed: end_date=X (want Y)", which is a value +#: mismatch rather than an absence. +_ABSENT_PATTERN = re.compile(r"\bno \d|\bno comments\b|\bnot archived\b|\bnot created\b") + def classify_failure( note: str | None, @@ -89,25 +97,30 @@ def classify_failure( Structural signals win over the note: a run that hit its ceiling has an unfinished state to report regardless of what the note says about it. """ - if hit_max_iterations or (stop_reason or "").strip().lower() in _CAPPED_STOP_REASONS: - return ABANDONED - text = (note or "").strip() - if not text: - return UNCLASSIFIED lowered = text.lower() + capped = hit_max_iterations or (stop_reason or "").strip().lower() in _CAPPED_STOP_REASONS if lowered.startswith("env:"): return ENVIRONMENT - # The verifier's own verdict outranks the prose after it: a note can say the - # answer was right and still be a failure, because the evidence was missing. - if _ANSWER_CORRECT in lowered: - return UNPROVEN + # A proven wrong answer outranks the cap. Running out of iterations explains why a + # run stopped, not why what it wrote was wrong, and calling that combination + # "abandoned" would file a demonstrated defect as a non-defect. + # + # The false marker is tested first because the true one is searched anywhere in the + # note, and a wrong value quoted back by the verifier can itself contain the string. if _ANSWER_WRONG in lowered: return WRONG_VALUE + if _ANSWER_CORRECT in lowered: + return UNPROVEN + + if capped: + return ABANDONED + if not text: + return UNCLASSIFIED - absent = any(marker in lowered for marker in _ABSENT) + absent = bool(_ABSENT_PATTERN.search(lowered)) or any(marker in lowered for marker in _ABSENT) present = any(marker in lowered for marker in _PRESENT) if absent and present: return PARTIAL_WRITE diff --git a/evals/core/pricing.py b/evals/core/pricing.py index 52618638..03a36233 100644 --- a/evals/core/pricing.py +++ b/evals/core/pricing.py @@ -120,12 +120,14 @@ def lookup_price(model_id: str | None) -> ModelPrice | None: def price_usage(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> RowCost: """Price one row's ``usage_total``.""" - if not usage_total or not has_token_counts(usage_total): - # A usage dict with no counts in it is metadata, not a measurement. - return RowCost(outcome=UNMEASURED, usd=None, model_id=model or None) - - vendor = usage_total.get("total_cost_usd") + vendor = usage_total.get("total_cost_usd") if usage_total else None vendor_usd = float(vendor) if isinstance(vendor, (int, float)) else None + + if not usage_total or not has_token_counts(usage_total): + # A usage dict with no counts in it is metadata, not a measurement -- but a vendor + # that stated what it charged still told us something, and discarding that would + # throw away the most authoritative figure available. + return RowCost(outcome=UNMEASURED, usd=None, model_id=model or None, vendor_usd=vendor_usd) model_id = resolve_model_id(usage_total, model=model) accounting = normalize_usage(usage_total, model=model_id) diff --git a/evals/core/results.py b/evals/core/results.py index 7e31b189..67090528 100644 --- a/evals/core/results.py +++ b/evals/core/results.py @@ -109,9 +109,10 @@ class CallRecord: raw_tool: str | None = None # Which kind of "no" an errored call received; None when the call succeeded. error_class: str | None = None - # The request body, recorded only under --record-result-payloads. Args are metrics-only - # by default (see args_chars); without the request, a recorded refusal can be read but - # not attributed to the target it names, which is the question a payload is kept to answer. + # The request body, recorded on every driver. Without it a recorded refusal can be read + # but not attributed to the target it names, which is the question it is kept to answer. + # Long string values are truncated (see ARGS_VALUE_LIMIT): the tool surface accepts rich + # HTML, page bodies and attachment URLs, so an untruncated copy is unbounded in size. args_json: str | None = None result_tokens_skipped: str | None = None # None means the response was not checked; [] means checked with no match. @@ -171,8 +172,8 @@ class TaskResult: 4 adds the task-local question fingerprint used by future intersection comparisons. Version 5 adds typed trace integrity and the observed tool-manifest fingerprint. Version 6 adds the reproducible per-repetition fixture seed id, non-secret fixture kinds, - and randomization namespaces. Target entity ids and randomized truth values are - deliberately excluded. + and randomization namespaces. Randomized truth values are deliberately excluded; request + arguments, which name the target a call acted on, are recorded with long values truncated. """ schema_version: int = RESULT_SCHEMA_VERSION @@ -530,6 +531,37 @@ def from_row(cls, row: dict[str, Any]) -> TaskResult: ) +#: Longest string value kept inside a recorded argument dict. Identifiers, actions and +#: enum values are far shorter than this, so what gets cut is prose: descriptions, HTML +#: bodies, comment text and query strings. A 1MB description_html would otherwise be copied +#: verbatim into the result file, which is neither useful for analysis nor safe to assume small. +ARGS_VALUE_LIMIT = 256 + +_TRUNCATION_MARK = "\u2026[truncated]" + + +def _bounded_args(args: dict[str, Any]) -> dict[str, Any]: + """Copy an argument dict with long string values cut to a fixed length. + + Structure is preserved so the target of a call stays legible; only bulk content + is dropped. Nested containers are bounded through their serialized form, since a + list of ids is worth keeping whole and a list of page bodies is not. + """ + bounded: dict[str, Any] = {} + for name, value in args.items(): + if isinstance(value, str) and len(value) > ARGS_VALUE_LIMIT: + bounded[name] = value[:ARGS_VALUE_LIMIT] + _TRUNCATION_MARK + elif isinstance(value, (list, tuple, dict)): + try: + encoded = json.dumps(value, default=str, ensure_ascii=False) + except Exception: + encoded = str(value) + bounded[name] = value if len(encoded) <= ARGS_VALUE_LIMIT else encoded[:ARGS_VALUE_LIMIT] + _TRUNCATION_MARK + else: + bounded[name] = value + return bounded + + def agent_run_to_task_result( run: AgentRun, ) -> TaskResult: @@ -605,9 +637,9 @@ def agent_run_to_task_result( # short strings. if isinstance(args, dict) and args: try: - rec.args_json = json.dumps(args, default=str, ensure_ascii=False) + rec.args_json = json.dumps(_bounded_args(args), default=str, ensure_ascii=False) except Exception: - rec.args_json = str(args) + rec.args_json = str(args)[:ARGS_VALUE_LIMIT] calls.append(rec) client_tool_calls: list[CallRecord] = [] diff --git a/evals/report/economics.py b/evals/report/economics.py index 4fc68950..67352f90 100644 --- a/evals/report/economics.py +++ b/evals/report/economics.py @@ -26,6 +26,23 @@ from .schema_friction import successful_trace_rows from .statistics import median, percentile + +def format_usd(amount: float) -> str: + """Render dollars without rounding a real figure down to nothing. + + A genuine $0.00032 printed as $0.000 is the same "reads as free" mistake the three + cost outcomes exist to prevent, so small non-zero amounts keep enough digits to stay + visible. Used for every figure, the drift diagnostic included. + """ + magnitude = abs(amount) + if magnitude < 1e-9: + # Float noise from summing many rows, not a real fraction of a cent. + return "$0.000" + if magnitude < 0.001: + return f"{'-' if amount < 0 else ''}${magnitude:.2e}" + return f"${amount:,.3f}" if amount >= 0 else f"-${magnitude:,.3f}" + + COST_LIMITATION = ( "limitation: cost is computed from a static price table; a model absent from it reports " "unpriced, and a row whose driver recorded no usage at all reports unmeasured. Neither is $0" @@ -62,6 +79,9 @@ class EconomicsMeasurement: unpriced_rows: int unmeasured_rows: int missing_input_rows: int + drift_computed_usd: float | None + drift_vendor_usd: float | None + drift_rows: int med_call_latency_ms: float | None p95_call_latency_ms: float | None prices_as_of: str = PRICES_AS_OF @@ -70,20 +90,20 @@ class EconomicsMeasurement: def cost_drift_usd(self) -> float | None: """How far the price table sits from what the vendor said it charged. - None when no vendor reported a figure, which is most runs. + Computed over the rows carrying *both* figures, so a row the vendor priced but the + table could not (or the reverse) adds coverage noise to neither side. None when no + row carries both, which is most runs. """ - if self.computed_cost_usd is None or self.vendor_cost_usd is None: + if self.drift_computed_usd is None or self.drift_vendor_usd is None: return None - return self.computed_cost_usd - self.vendor_cost_usd + return self.drift_computed_usd - self.drift_vendor_usd @property def cost_text(self) -> str: """Never render an unknown cost as a number, or a real one as zero.""" if self.cost_usd is None: return UNMEASURED if self.cost_outcome == UNMEASURED else UNPRICED - # A run that really cost a fraction of a cent must not print $0.000; that is - # the same "reads as free" mistake in a different disguise. - text = "<$0.001" if 0 < self.cost_usd < 0.001 else f"${self.cost_usd:,.3f}" + text = format_usd(self.cost_usd) if self.unpriced_rows or self.unmeasured_rows: text += f" (+{self.unpriced_rows} unpriced, {self.unmeasured_rows} unmeasured rows)" return text @@ -110,10 +130,14 @@ def _charged_rows(rows: list[ResultRow]) -> list[TaskResult]: charged: list[TaskResult] = [] for raw_row in rows: row = read_result(raw_row) - if is_meta_row(row) or is_infra_error_row(row): - continue - if (row.error or row.skipped) and not has_token_counts(row.usage_total): + if is_meta_row(row): continue + if row.error or row.skipped or is_infra_error_row(row): + # An infrastructure classification is applied *after* the agent run is folded + # in, so a contained CLI timeout or trace failure can carry real usage. Keep + # any row that shows the model ran; drop only ones that never started. + if not has_token_counts(row.usage_total) and not row.calls: + continue charged.append(row) return charged @@ -140,6 +164,9 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: saw_computed = False vendor_total = 0.0 saw_vendor = False + drift_computed = 0.0 + drift_vendor = 0.0 + drift_rows = 0 priced = unpriced = unmeasured = 0 for row in executed: tokens = _row_input_tokens(row) @@ -168,6 +195,13 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: if cost.vendor_usd is not None: vendor_total += cost.vendor_usd saw_vendor = True + # Drift is only meaningful over rows that carry *both* figures. Summing each side + # independently would fold coverage differences into what is meant to be a + # price-table comparison. + if cost.usd is not None and cost.vendor_usd is not None: + drift_computed += cost.usd + drift_vendor += cost.vendor_usd + drift_rows += 1 if not saw_billed: # Nothing to report: say which kind of nothing it is. @@ -205,6 +239,9 @@ def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: computed_cost_usd=computed_total if saw_computed else None, vendor_cost_usd=vendor_total if saw_vendor else None, missing_input_rows=missing_input, + drift_computed_usd=drift_computed if drift_rows else None, + drift_vendor_usd=drift_vendor if drift_rows else None, + drift_rows=drift_rows, cost_outcome=outcome, priced_rows=priced, unpriced_rows=unpriced, @@ -226,14 +263,15 @@ def economics_statement(measurement: EconomicsMeasurement) -> str: f" wall time={measurement.total_wall_time_s:,.0f}s; call latency {latency_text}", ] drift = measurement.cost_drift_usd - if drift is not None and measurement.vendor_cost_usd is not None: - # The only standing check that the price table has not gone stale, so it has to - # compare the table's own figure against the vendor's -- not the reported cost, - # which already prefers the vendor and would always agree with itself. + if drift is not None: + # The only standing check that the price table has not gone stale, so it compares + # the table's own figure against the vendor's -- not the reported cost, which + # already prefers the vendor and would always agree with itself. lines.append( - f" vendor-reported cost=${measurement.vendor_cost_usd:,.3f}; " - f"price table computes ${measurement.computed_cost_usd:,.3f} " - f"(differs by ${drift:+,.3f})" + f" price-table check over {measurement.drift_rows} row(s) carrying both: " + f"vendor {format_usd(measurement.drift_vendor_usd)}, " + f"table {format_usd(measurement.drift_computed_usd)} " + f"(differs by {format_usd(drift)})" ) lines.append(f" {COST_LIMITATION}") return "\n".join(lines) @@ -241,6 +279,7 @@ def economics_statement(measurement: EconomicsMeasurement) -> str: __all__ = [ "COST_LIMITATION", + "format_usd", "EconomicsMeasurement", "TaskEconomics", "economics_statement", diff --git a/evals/report/lookup_reuse.py b/evals/report/lookup_reuse.py index 02446b7c..9c070306 100644 --- a/evals/report/lookup_reuse.py +++ b/evals/report/lookup_reuse.py @@ -1,4 +1,4 @@ -"""How often an agent re-hunts an entity whose identifier it already holds. +"""How often an agent goes looking for an entity it has already resolved. The sharpest single finding of the 2026-08-24 cross-harness pair was ``workitem.search`` 111 vs 26 against ``workitem.retrieve`` 4 vs 20. One arm carried @@ -6,9 +6,21 @@ property of the **surface** as much as of the agent -- identifiers that stayed sticky would close the gap without either agent changing -- and nothing measured it. -Read from recorded call arguments, so it is only answerable on runs that have them. -A run that does not is reported as not measured, never as zero: "no redundant -lookups" and "we could not tell" are opposite conclusions. +The rule is deliberately narrow, because the first version was not and counted +things no reasonable reader would call redundant: + + same entity only the resource's *own* identifier counts. ``project_id`` on a + work item call says where to look, not which item is known, and + treating it as "in hand" flagged every list in a project. + not paging a lookup carrying a cursor or offset is continuing one traversal, + not starting a second. + args only ids are read from request arguments. An id that arrived in a + *result* is invisible here, so this undercounts rather than over. + +What survives is still a heuristic: retrieving item A and then searching for an +unrelated item B of the same resource is counted, because nothing in the arguments +distinguishes that from re-hunting A. Read it as an upper bound on identifier +stickiness, not as a defect count. """ from __future__ import annotations @@ -19,14 +31,19 @@ from evals.core.results import CallRecord, TaskResult -from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .load import ResultRow, is_meta_row, read_result + +LOOKUP_REUSE_LIMITATION = ( + "limitation: counts a search/list on a resource whose own id was already an argument, " + "excluding paginated continuations. It cannot tell re-hunting the same entity from looking " + "up a different one of the same kind, and ids that arrived only in results are invisible" +) #: Actions that go looking for something rather than addressing it directly. -_SEARCH_ACTIONS = frozenset({"search", "list"}) +_SEARCH_ACTIONS = frozenset({"search", "list", "list_archived"}) -#: Argument names that carry an identifier this run already resolved. Matched by -#: suffix so ``workitem_id``, ``parent_id`` and ``id`` all count. -_ID_SUFFIX = "_id" +#: Arguments that mean "continue the previous traversal" rather than "look again". +_PAGINATION_ARGS = frozenset({"cursor", "offset", "page", "next_cursor", "page_token"}) def _args_of(call: CallRecord) -> dict | None: @@ -39,14 +56,14 @@ def _args_of(call: CallRecord) -> dict | None: return parsed if isinstance(parsed, dict) else None -def _ids_in(args: dict) -> set[str]: - found: set[str] = set() - for name, value in args.items(): - if not isinstance(value, str) or not value: - continue - if name == "id" or name.endswith(_ID_SUFFIX): - found.add(value) - return found +def _own_ids(resource: str, args: dict) -> set[str]: + """Identifiers of *this* resource, ignoring scope and cross-references. + + ``workitem_id`` on the ``workitem`` tool is the entity; ``project_id`` is the + scope it lives in, and ``cycle_id`` is a different resource entirely. + """ + own = {"id", f"{resource}_id"} + return {value for name, value in args.items() if name in own and isinstance(value, str) and value} @dataclass(frozen=True, slots=True) @@ -57,6 +74,7 @@ class LookupReuseMeasurement: by_resource: dict[str, int] = field(default_factory=dict) rows_measured: int = 0 rows_without_args: int = 0 + calls_without_args: int = 0 @property def measurable(self) -> bool: @@ -66,16 +84,21 @@ def statement(self) -> str: if not self.measurable: return f"redundant lookups: not measured — {self.rows_without_args} row(s) carry no recorded call arguments" detail = ", ".join(f"{resource}={count}" for resource, count in sorted(self.by_resource.items())) - line = f"redundant lookups: {self.total} (a search or list on a resource whose id was already in hand)" + line = f"redundant lookups: {self.total}" if detail: line += f" [{detail}]" + extras = [] if self.rows_without_args: - line += f"; {self.rows_without_args} row(s) not measured for want of arguments" - return line + extras.append(f"{self.rows_without_args} row(s) not measured for want of arguments") + if self.calls_without_args: + extras.append(f"{self.calls_without_args} call(s) skipped inside measured rows") + if extras: + line += "; " + "; ".join(extras) + return f"{line}\n {LOOKUP_REUSE_LIMITATION}" def measure_lookup_reuse(rows: list[ResultRow]) -> LookupReuseMeasurement: - """Count searches issued after the same resource's id was already resolved. + """Count searches issued after the same resource's own id was already an argument. Scoped to one row. Each repetition is a fresh conversation, so an id learned in one tells the agent in another nothing. @@ -83,34 +106,38 @@ def measure_lookup_reuse(rows: list[ResultRow]) -> LookupReuseMeasurement: total = 0 by_resource: dict[str, int] = defaultdict(int) measured = 0 - without_args = 0 + rows_without_args = 0 + calls_without_args = 0 for raw_row in rows: row: TaskResult = read_result(raw_row) - if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + if is_meta_row(row): continue if not any(call.args_json for call in row.calls): - without_args += 1 + rows_without_args += 1 continue measured += 1 known: dict[str, set[str]] = defaultdict(set) for call in row.calls: args = _args_of(call) if args is None: + calls_without_args += 1 continue resource = call.tool action = (call.action or "").lower() - if action in _SEARCH_ACTIONS and known[resource]: + paging = any(name in args for name in _PAGINATION_ARGS) + if action in _SEARCH_ACTIONS and known[resource] and not paging: total += 1 by_resource[resource] += 1 # Learned after the check, so the call that first resolves an id is never # charged for the lookup that produced it. - known[resource].update(_ids_in(args)) + known[resource].update(_own_ids(resource, args)) return LookupReuseMeasurement( total=total, by_resource=dict(by_resource), rows_measured=measured, - rows_without_args=without_args, + rows_without_args=rows_without_args, + calls_without_args=calls_without_args, ) -__all__ = ["LookupReuseMeasurement", "measure_lookup_reuse"] +__all__ = ["LOOKUP_REUSE_LIMITATION", "LookupReuseMeasurement", "measure_lookup_reuse"] diff --git a/evals/report/power.py b/evals/report/power.py index ab5fa9cd..ff0f9f3a 100644 --- a/evals/report/power.py +++ b/evals/report/power.py @@ -25,20 +25,25 @@ def power_statement(summary: Summary) -> str | None: """Return the guardrail line, or None when at least one task is well powered. - The claim is deliberately about the best-covered task: if any task reaches the - threshold, "no per-task verdict is supported" would be false, and a caveat that - overstates its own scope gets ignored along with the ones that do not. + Scoped to the tasks that are actually shallow. An earlier version keyed off the + best-covered task and fell silent as soon as any one task was deep, which left a + mixed run's shallow tasks uncaveated -- the opposite of the intended failure. + + The threshold is a reporting heuristic, not a power calculation: 5/5 still carries a + Wilson interval of roughly [0.57, 1.00], so the line points readers at the aggregate + rather than promising that five repetitions settle anything. """ counts = [task.n for task in summary.tasks.values() if task.n] if not counts: return None - best = max(counts) - if best >= UNDERPOWERED_REPS: + shallow = [count for count in counts if count < UNDERPOWERED_REPS] + if not shallow: return None return ( - f"POWER: {best} repetition(s) per task at most — per-task pass rates and UNSTABLE flags " - f"are not verdicts at this depth; read the aggregate and paired deltas instead. " - f"Use --tasks with --reps {UNDERPOWERED_REPS}+ for a per-task claim." + f"POWER: {len(shallow)} of {len(counts)} task(s) below {UNDERPOWERED_REPS} repetitions " + f"(fewest {min(shallow)}) — their per-task pass rates and UNSTABLE flags are not verdicts " + f"at that depth; read the aggregate and paired deltas for those. Raising --reps narrows " + f"a per-task interval but no fixed count makes one conclusive." ) diff --git a/tests/evals/report/test_power.py b/tests/evals/report/test_power.py index 3ab21fa8..387030e2 100644 --- a/tests/evals/report/test_power.py +++ b/tests/evals/report/test_power.py @@ -37,17 +37,23 @@ def test_the_guardrail_is_absent_at_five_reps(): assert power_statement(summarize(rows_at(UNDERPOWERED_REPS))) is None -def test_a_single_well_powered_task_suppresses_the_blanket_claim(): - """The line says no task is well powered, so one that is makes it false.""" +def test_one_deep_task_does_not_silence_the_caveat_for_the_shallow_ones(): + """Keying off the best-covered task left a mixed run's shallow tasks uncaveated. + + That is the opposite of the intended failure, so the line is scoped to the tasks + that are actually shallow and names how many they are. + """ rows = rows_at(2, tasks=2) + [ {"task_id": "T9", "rep": rep, "success": True, "trace_integrity": True, "num_calls": 1, "calls": []} for rep in range(UNDERPOWERED_REPS) ] - assert power_statement(summarize(rows)) is None + statement = power_statement(summarize(rows)) + assert statement is not None + assert "2 of 3" in statement def test_the_guardrail_names_the_rep_count_it_saw(): - assert "2" in (power_statement(summarize(rows_at(2))) or "") + assert "fewest 2" in (power_statement(summarize(rows_at(2))) or "") def test_no_evaluated_rows_produces_no_claim(): diff --git a/tests/evals/report/test_review_round2.py b/tests/evals/report/test_review_round2.py new file mode 100644 index 00000000..4d427dc9 --- /dev/null +++ b/tests/evals/report/test_review_round2.py @@ -0,0 +1,222 @@ +"""Regressions from the second adversarial review. + +Ten findings. The two that mattered most were a metric measuring something other +than what it claimed, and a field documented as "ids and short strings" that had no +size bound at all. +""" + +from __future__ import annotations + +import json + +from evals.core.failure_kind import ABANDONED, MISSING_WRITE, PARTIAL_WRITE, UNPROVEN, WRONG_VALUE, classify_failure +from evals.core.pricing import price_usage +from evals.core.results import AgentRun, Usage, agent_run_to_task_result +from evals.report import summarize +from evals.report.economics import measure_economics +from evals.report.lookup_reuse import measure_lookup_reuse +from evals.report.power import power_statement + + +def call(tool, action, **args): + return {"tool": tool, "action": action, "args_json": json.dumps(args)} + + +def task_row(*calls, **overrides): + row = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": len(calls), + "calls": list(calls), + } + row.update(overrides) + return row + + +# --- F1: the lookup rule must establish reuse of the same entity ------------------ + + +def test_f1_pagination_is_not_a_redundant_lookup(): + assert ( + measure_lookup_reuse( + [ + task_row( + call("workitem", "list", project_id="p", cursor="a"), + call("workitem", "list", project_id="p", cursor="b"), + ) + ] + ).total + == 0 + ) + + +def test_f1_a_list_after_a_create_is_not_a_redundant_lookup(): + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "create", project_id="p", name="x"), call("workitem", "list", project_id="p"))] + ).total + == 0 + ) + + +def test_f1_a_scope_id_does_not_put_the_entity_in_hand(): + """project_id says which project to look in, not which work item is known.""" + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "retrieve", project_id="p"), call("workitem", "search", query="z"))] + ).total + == 0 + ) + + +def test_f1_the_entitys_own_id_still_counts(): + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "retrieve", workitem_id="wi-1"), call("workitem", "search", query="z"))] + ).total + == 1 + ) + + +# --- F2: recorded arguments need a size bound ------------------------------------ + + +def test_f2_a_huge_argument_value_is_bounded(): + run = AgentRun( + calls=[{"tool": "workitem", "args": {"action": "update", "description_html": "x" * 1_000_000}}], + final_text="", + usage=Usage(), + stopped_reason="end_turn", + call_source="api", + ) + args_json = agent_run_to_task_result(run).calls[0].args_json + assert args_json is not None + assert len(args_json) < 5_000, f"args_json was {len(args_json)} bytes" + parsed = json.loads(args_json) + # Structure and the short discriminating values survive; only the bulk is cut. + assert parsed["action"] == "update" + assert parsed["description_html"].endswith("…[truncated]") + + +def test_f2_short_arguments_are_untouched(): + args = {"action": "retrieve", "workitem_id": "wi-42"} + run = AgentRun( + calls=[{"tool": "workitem", "args": args}], + final_text="", + usage=Usage(), + stopped_reason="end_turn", + call_source="api", + ) + assert json.loads(agent_run_to_task_result(run).calls[0].args_json) == args + + +# --- F3: charged rows the runner later marked infrastructure --------------------- + + +def test_f3_an_infra_terminated_row_that_burned_tokens_is_still_charged(): + usage = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 0, "source": "iterations"} + rows = [task_row(model="gpt-5.6-luna", usage_total=usage, error="infra_cli timeout", error_class="infra_cli")] + measurement = measure_economics(rows) + assert measurement.priced_rows == 1 + assert measurement.cost_usd is not None + + +def test_f3_a_vendor_only_cost_is_not_thrown_away(): + """Tokens unmeasured, but the vendor still said what it charged.""" + cost = price_usage({"source": "modelUsage", "total_cost_usd": 1.25}, model="haiku") + assert cost.vendor_usd == 1.25 + assert cost.billed_usd == 1.25 + + +# --- F5/F6: classifier precedence ------------------------------------------------- + + +def test_f5_a_wrong_answer_quoting_the_true_marker_is_not_unproven(): + note = "answer_correct=false (values=[\"answer_correct=true\"]; want ['x'])" + assert classify_failure(note) == WRONG_VALUE + + +def test_f6_a_capped_run_that_also_wrote_the_wrong_value_is_not_a_non_defect(): + """A cap can coexist with a proven wrong write; abandoned would excuse it.""" + assert classify_failure("answer_correct=false (values=['a']; want ['b'])", hit_max_iterations=True) == WRONG_VALUE + + +def test_f6_a_capped_run_with_an_unfinished_state_is_still_abandoned(): + assert classify_failure("estimate points missing fib subset", hit_max_iterations=True) == ABANDONED + + +def test_f6_linked_counts_as_a_present_marker(): + assert classify_failure("request 'x' missing; R1 item ABC-1 linked") == PARTIAL_WRITE + + +def test_f6_absence_phrasings_from_real_verifiers_are_recognised(): + for note in ("no comments on target item", "no 120-minute work log", "2 module items not archived"): + assert classify_failure(note) == MISSING_WRITE, note + + +def test_unproven_still_works(): + assert classify_failure("answer_correct=true (...); provenance=missing (0 evidence-bearing)") == UNPROVEN + + +# --- F7: the power line must not be silenced by one well-covered task ------------- + + +def test_f7_underpowered_tasks_are_still_named_when_another_task_is_deep(): + rows = [task_row(task_id="T1", rep=rep, success=True, calls=[]) for rep in range(2)] + [ + task_row(task_id="T2", rep=rep, success=True, calls=[]) for rep in range(5) + ] + statement = power_statement(summarize(rows)) + assert statement is not None + assert "1 of 2" in statement + + +def test_f7_no_line_when_every_task_is_deep_enough(): + rows = [task_row(task_id="T1", rep=rep, success=True, calls=[]) for rep in range(5)] + assert power_statement(summarize(rows)) is None + + +# --- F8/F10: the drift cohort and its formatting --------------------------------- + + +def test_f8_drift_uses_only_rows_carrying_both_figures(): + both = task_row( + model="haiku", + usage_total={ + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + }, + ) + computed_only = task_row( + task_id="R2", + model="gpt-5.6-luna", + usage_total={"input_tokens": 900_000, "output_tokens": 5000, "source": "iterations"}, + ) + measurement = measure_economics([both, computed_only]) + assert measurement.drift_rows == 1 + # Without cohort alignment the second row's cost would inflate the drift. + assert abs(measurement.cost_drift_usd) < 0.01 + + +def test_f10_a_sub_cent_drift_line_does_not_read_as_all_zeroes(): + tiny = task_row( + model="haiku", + usage_total={ + "input_tokens": 100, + "output_tokens": 1, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 100, + "total_cost_usd": 0.00032, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + }, + ) + from evals.report.economics import economics_statement + + line = economics_statement(measure_economics([tiny])) + assert "$0.000;" not in line and "$0.000 " not in line diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py index da740282..02fc016f 100644 --- a/tests/evals/report/test_summary.py +++ b/tests/evals/report/test_summary.py @@ -411,9 +411,10 @@ def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): " limitation: a first not_found is read as the answer to an existence question, since asking has no " "cheaper form; only a repeat on the same tool and action is counted as friction. A surface that " "misleads an agent into one wrong lookup is therefore not charged for it\n" - "POWER: 1 repetition(s) per task at most \u2014 per-task pass rates and UNSTABLE flags " - "are not verdicts at this depth; read the aggregate and paired deltas instead. " - "Use --tasks with --reps 5+ for a per-task claim.\n" + "POWER: 1 of 1 task(s) below 5 repetitions (fewest 1) \u2014 their per-task pass rates " + "and UNSTABLE flags are not verdicts at that depth; read the aggregate and paired " + "deltas for those. Raising --reps narrows a per-task interval but no fixed count " + "makes one conclusive.\n" "failure kinds: no failed rows\n" "redundant lookups: not measured \u2014 1 row(s) carry no recorded call arguments\n" "economics: cost=unmeasured (prices as of 2026-08-24); input tokens=unmeasured; result tokens=0\n" From 35e91b9c88332d0e3a00356fe77a03bb460bfd6d Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Ketireddy Date: Tue, 25 Aug 2026 09:57:35 +0530 Subject: [PATCH 93/93] fix(tests): bind SDK signatures that Python 3.14 will not evaluate 17 tests across four files failed with "'function' object is not subscriptable", none of them about annotations. plane-sdk declares twelve methods as `def list(self, ...) -> list[...]`. Python 3.14 made annotations lazy (PEP 649) and inspect.signature evaluates them, and in a class body the class namespace is in scope by then -- so `list` resolves to the method being defined rather than the builtin, and subscripting a function raises. Before 3.14 the annotation was evaluated at def time, before that name was bound, so it never came up. get_type_hints was already guarded for this; the two inspect.signature calls were not. Both now go through _signature_of, which falls back to the FORWARDREF annotation format. That still resolves what it can, so binding and type-checking both survive rather than degrading to strings, and any method that needed the fallback is recorded in UNEVALUATED_ANNOTATIONS -- a type-check that quietly checks nothing is worse than one that fails. The new test reproduces the shadowing shape locally, so it holds without pinning an SDK version and passes on interpreters that never hit the problem. Nothing was broken at runtime: the server advertises all 28 tools and a full four-arm eval battery ran 280 rows clean. This is test-time introspection only, it predates this branch, and no CI workflow runs pytest. The real fix belongs upstream -- those twelve annotations want typing.List or builtins.list, or the methods want renaming. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tools/_spyclient.py | 34 ++++++++++++++- tests/tools/test_governance.py | 4 +- tests/tools/test_spyclient_signatures.py | 54 ++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 tests/tools/test_spyclient_signatures.py diff --git a/tests/tools/_spyclient.py b/tests/tools/_spyclient.py index 955803f6..db60f300 100644 --- a/tests/tools/_spyclient.py +++ b/tests/tools/_spyclient.py @@ -20,6 +20,11 @@ from plane import PlaneClient from pydantic import BaseModel, TypeAdapter +try: # Python 3.14+ evaluates annotations inside inspect.signature; see _signature_of + from annotationlib import Format as _AnnotationFormat +except ImportError: # pragma: no cover - Python < 3.13 has no annotationlib + _AnnotationFormat = None + types_UnionType = type(int | str) # `X | Y` annotations are not typing.Union @@ -132,12 +137,39 @@ def _validate(method: str, param: inspect.Parameter, annotation: Any, value: Any raise TypeError(f"{method}(): argument {param.name}={value!r} does not satisfy {annotation}: {exc}") from exc +#: SDK methods whose annotations would not evaluate eagerly. Kept visible rather than +#: swallowed: a type-check that quietly checks nothing is worse than one that fails. +UNEVALUATED_ANNOTATIONS: set[str] = set() + + +def _signature_of(path: str, fn: Any) -> inspect.Signature: + """Bind-capable signature, even for a method whose annotations will not evaluate. + + Python 3.14 (PEP 649) made annotations lazy, and ``inspect.signature`` evaluates + them. plane-sdk declares twelve methods as ``def list(self, ...) -> list[...]``, + and under lazy evaluation the class namespace is in scope, so ``list`` resolves to + the method being defined rather than the builtin -- subscripting a function raises + TypeError. Before 3.14 the annotation was evaluated at ``def`` time, before that + name was bound, so this never came up. + + FORWARDREF still resolves what it can (``list[int]`` comes back intact) and leaves + the rest as ForwardRefs, so binding *and* type-checking survive the fallback. + """ + try: + return inspect.signature(fn) + except (TypeError, NameError): + if _AnnotationFormat is None: # pragma: no cover - Python < 3.14 never gets here + raise + UNEVALUATED_ANNOTATIONS.add(path) + return inspect.signature(fn, annotation_format=_AnnotationFormat.FORWARDREF) + + class _Method: def __init__(self, spy: SpyClient, path: str, fn: Any) -> None: self._spy = spy self._path = path self._fn = fn - self._signature = inspect.signature(fn) + self._signature = _signature_of(path, fn) try: self._hints = get_type_hints(fn) except Exception: diff --git a/tests/tools/test_governance.py b/tests/tools/test_governance.py index 1e5521e7..27972baa 100644 --- a/tests/tools/test_governance.py +++ b/tests/tools/test_governance.py @@ -14,13 +14,13 @@ from __future__ import annotations -import inspect from types import SimpleNamespace import pytest from plane.errors.errors import HttpError from plane_mcp.tools.workitem_type import _scope_of +from tests.tools._spyclient import _signature_of PROJECT = "project-1" TYPE_ID = "type-1" @@ -85,7 +85,7 @@ def test_the_resolver_matches_the_sdk(project_id): for verb in ("list", "retrieve", "create", "update", "delete"): method = getattr(namespace, verb, None) assert method is not None, f"the SDK namespace has no {verb}()" - takes = inspect.signature(method).parameters + takes = _signature_of(f"{verb}", method).parameters if verb in ("retrieve", "update", "delete"): assert id_kwarg in takes, f"{verb}() does not take {id_kwarg!r}" for name in scope: diff --git a/tests/tools/test_spyclient_signatures.py b/tests/tools/test_spyclient_signatures.py new file mode 100644 index 00000000..72b3e260 --- /dev/null +++ b/tests/tools/test_spyclient_signatures.py @@ -0,0 +1,54 @@ +"""The spy must bind against an SDK method whose annotations will not evaluate. + +plane-sdk declares twelve methods as ``def list(self, ...) -> list[...]``. Under +Python 3.14's lazy annotations (PEP 649) the class namespace is in scope when the +annotation is evaluated, so ``list`` resolves to the method rather than the builtin +and subscripting it raises TypeError. That took out 17 tests across four files -- +none of which are about annotations -- because ``inspect.signature`` does the +evaluating. + +Nothing fails at runtime: the server advertises all 28 tools and a full eval battery +runs clean. The breakage is confined to test-time introspection. +""" + +import inspect + +from tests.tools._spyclient import UNEVALUATED_ANNOTATIONS, _signature_of + + +class ShadowingResource: + """The exact shape plane-sdk uses, reproduced so this test needs no SDK version.""" + + def list(self, workspace_slug: str) -> list[int]: + return [] + + +def test_the_shadowing_pattern_still_yields_a_bindable_signature(): + signature = _signature_of("shadow.list", ShadowingResource.list) + assert list(signature.parameters) == ["self", "workspace_slug"] + bound = signature.bind(ShadowingResource(), workspace_slug="acme") + assert bound.arguments["workspace_slug"] == "acme" + + +def test_a_degraded_signature_is_recorded_rather_than_silently_accepted(): + """A type-check that quietly checks nothing must not look like a passing one.""" + try: + inspect.signature(ShadowingResource.list) + except TypeError: + # This interpreter evaluates annotations eagerly here, so the fallback ran. + _signature_of("shadow.list", ShadowingResource.list) + assert "shadow.list" in UNEVALUATED_ANNOTATIONS + else: + # Pre-3.14: no fallback needed, so nothing should be recorded for it. + _signature_of("shadow.list", ShadowingResource.list) + assert "shadow.list" not in UNEVALUATED_ANNOTATIONS + + +def test_an_ordinary_signature_is_untouched(): + def plain(a: int, b: str = "x") -> bool: + return True + + signature = _signature_of("plain", plain) + assert list(signature.parameters) == ["a", "b"] + assert signature.parameters["a"].annotation is int + assert "plain" not in UNEVALUATED_ANNOTATIONS