diff --git a/CLAUDE.md b/CLAUDE.md index afa6138..3d0eee9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -185,10 +185,14 @@ Configs: - **Do not `source .env`; let `cb` load it or parse ad-hoc checks with `python-dotenv`.** **Why:** `.env` is dotenv syntax rather than guaranteed shell syntax, and currently contains an unquoted value with spaces that a shell tries to execute as a command. -- **Patient simulation on `claude-sonnet-5` must explicitly disable adaptive thinking unless its - response parser and token budget are redesigned for thinking blocks.** **Why:** Sonnet 5 enables - thinking by default, while this simulator expects a text-first response within 1,024 output - tokens; `thinking={"type": "disabled"}` also requires `anthropic>=0.101.0`. +- **Both patient and P2P counterpart simulation on `claude-sonnet-5` must explicitly disable + adaptive thinking unless their response parsers and token budgets are redesigned for thinking + blocks.** **Why:** Sonnet 5 enables thinking by default; both simulators read the first block's + `.text`, with budgets of 1,024 and 512 output tokens respectively. Updating only the patient + call leaves P2P `send_turn` failing on `ThinkingBlock`; `thinking={"type": "disabled"}` also + requires `anthropic>=0.101.0`. + Treat an empty P2P model override as the default too: Harbor Hub export forwards an omitted + override as an empty string, so testing only an absent environment variable misses that path. - **Do not pass Harbor's `thinking` agent kwarg to the stock `claude-code` harness without first checking the pinned Claude Code CLI.** **Why:** Harbor 0.6.1 renders it as `--thinking`, but Claude Code 2.1.207 and 2.1.216 do not expose that flag; use `reasoning_effort`/`--effort` diff --git a/README.md b/README.md index e43c368..e341459 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,11 @@ and image digest with your results. See [the v1.1.0 changes](CHANGELOG.md). The image bundles the FastAPI server, the workspace judge, the agent harness, and per-task fixtures. +P2P counterpart simulation defaults to `claude-sonnet-5` when `CHI_BENCH_P2P_SIMULATOR_MODEL` +is unset or empty. A non-empty value overrides it, and the `sonnet` alias selects the same model. +Both P2P and patient simulation disable adaptive thinking for `claude-sonnet-5` to preserve +text replies within their token budgets. Rebuild the image after updating simulator code. + **Verify setup:** ```bash diff --git a/src/chi_bench/services/p2p.py b/src/chi_bench/services/p2p.py index 1790c51..53e5125 100644 --- a/src/chi_bench/services/p2p.py +++ b/src/chi_bench/services/p2p.py @@ -529,9 +529,9 @@ def _generate_counterpart_reply( import anthropic - model_env = os.environ.get("CHI_BENCH_P2P_SIMULATOR_MODEL", "sonnet") + model_env = os.environ.get("CHI_BENCH_P2P_SIMULATOR_MODEL") or "sonnet" model_map = { - "sonnet": "claude-sonnet-4-20250514", + "sonnet": "claude-sonnet-5", "opus": "claude-opus-4-20250514", "haiku": "claude-haiku-4-5-20251001", } @@ -549,11 +549,17 @@ def _generate_counterpart_reply( messages.append({"role": "user", "content": message}) client = anthropic.Anthropic() + request_options: dict[str, Any] = {} + if model == "claude-sonnet-5": + # Keep the text-only reply shape and 512-token budget, as in + # the patient simulator, when adaptive thinking defaults on. + request_options["thinking"] = {"type": "disabled"} response = client.messages.create( model=model, max_tokens=512, system=counterpart_context, messages=messages, + **request_options, ) reply_text = response.content[0].text.strip() return reply_text, None diff --git a/tests/unit/test_p2p_simulator.py b/tests/unit/test_p2p_simulator.py new file mode 100644 index 0000000..744d511 --- /dev/null +++ b/tests/unit/test_p2p_simulator.py @@ -0,0 +1,98 @@ +import json +from datetime import UTC, datetime + +import anthropic +import httpx +import pytest + +from chi_bench.core.models import P2PSessionRecord, PayerPeerToPeerRequest +from chi_bench.services.p2p import P2PService + + +@pytest.mark.parametrize("benchmark_side", ["payer", "provider"]) +@pytest.mark.parametrize( + ("model_env", "expected_model"), + [ + (None, "claude-sonnet-5"), + ("", "claude-sonnet-5"), + ("sonnet", "claude-sonnet-5"), + ("claude-sonnet-5", "claude-sonnet-5"), + ("haiku", "claude-haiku-4-5-20251001"), + ], +) +def test_p2p_simulator_returns_text(monkeypatch, benchmark_side, model_env, expected_model): + if model_env is None: + monkeypatch.delenv("CHI_BENCH_P2P_SIMULATOR_MODEL", raising=False) + else: + monkeypatch.setenv("CHI_BENCH_P2P_SIMULATOR_MODEL", model_env) + + requests = [] + + def respond(request): + payload = json.loads(request.content) + requests.append(payload) + if payload["model"] == "claude-sonnet-4-20250514": + return httpx.Response( + 404, + json={ + "type": "error", + "error": { + "type": "not_found_error", + "message": "model: claude-sonnet-4-20250514", + }, + }, + ) + content = [{"type": "text", "text": " Let's review the evidence. "}] + if payload["model"] == "claude-sonnet-5" and payload.get("thinking") != { + "type": "disabled" + }: + content.insert(0, {"type": "thinking", "thinking": "Review", "signature": "test"}) + return httpx.Response( + 200, + json={ + "id": "msg_p2p", + "type": "message", + "role": "assistant", + "model": payload["model"], + "content": content, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 8}, + }, + ) + + now = datetime(2026, 10, 7, tzinfo=UTC) + p2p_request = PayerPeerToPeerRequest( + id="p2p-1", case_id="case-1", created_at=now, updated_at=now + ) + session = P2PSessionRecord( + id="session-1", + request_id=p2p_request.id, + case_id=p2p_request.case_id, + benchmark_side=benchmark_side, + created_at=now, + updated_at=now, + ) + service = P2PService(ctx=None) + monkeypatch.setattr(service, "_build_provider_context", lambda request: "You are the provider.") + monkeypatch.setattr(service, "_build_payer_context", lambda request: "You are the payer.") + with anthropic.Anthropic( + api_key="test-key", http_client=httpx.Client(transport=httpx.MockTransport(respond)) + ) as client: + monkeypatch.setattr(anthropic, "Anthropic", lambda: client) + reply, revealed_fact_id = service._generate_counterpart_reply( + session=session, + request=p2p_request, + contract={}, + message="Can we review the evidence?", + agent_turn_index=1, + ) + + assert reply == "Let's review the evidence." + assert revealed_fact_id is None + assert len(requests) == 1 + assert requests[0]["model"] == expected_model + if expected_model == "claude-sonnet-5": + assert requests[0]["thinking"] == {"type": "disabled"} + else: + assert "thinking" not in requests[0]