CHI-Bench: HG-Market-System repaired v1.1.0 packet · 20/75 · 26.67%

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/README.md

@@ -0,0 +1,14 @@
+# CHI-Bench submission: hg-chi-full75-20261006
+
+- Overall: **20/75 = 26.67%**
+- Provider PA: **11/25 = 44.00%**
+- Payer UM: **7/25 = 28.00%**
+- Care Management: **2/25 = 8.00%**
+
+This packet began as the completed Full75 and was then repaired only for the four evidence points explicitly requested by the CHI-Bench maintainer.
+
+- Care `cm_mdd_hard_refuses_002`: v1.1.0 judge-only re-evaluation from the preserved final state returned **0/5** judge rubrics, so the historical pass is corrected to a fail.
+- Payer P2P `t016`, `t019`, `t036`: exact v1.1.0 reruns with the historical Full75 HG agent snapshot and `CHI_BENCH_P2P_SIMULATOR_MODEL=claude-sonnet-5` all remained **1.0/1.0**.
+- The other **71** trial artifacts remain unchanged.
+
+See `repair_evidence.json` for run IDs, artifact digests, task-level hashes and the exact repair provenance.

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/provenance.json

@@ -0,0 +1,32 @@
+{
+  "chi_bench_git_sha": "c05b225bbb408f78a3043b7d40f022255bd38b19",
+  "image_digest": null,
+  "judge_model": "claude-opus-4-7",
+  "judge_num_votes": 1,
+  "harness_version": "0.1.0",
+  "hg_source_runs": [
+    {
+      "run_id": 37390621096,
+      "head_sha": "d0d7f72c23a6bf71b325e6cf373c40319ff4963a",
+      "scope": "25 provider + 25 payer + first 15 verified care"
+    },
+    {
+      "run_id": 37444743430,
+      "head_sha": "226e47675520a4408a014edbf8483f64024007ba",
+      "scope": "exact remaining 10 care tasks"
+    },
+    {
+      "run_id": 37625832254,
+      "head_sha": "36e19b84b64ca59c30acf933de612a04e8f37a16",
+      "scope": "Care judge-only repair for cm_mdd_hard_refuses_002 from preserved final state"
+    },
+    {
+      "run_id": 37628570479,
+      "head_sha": "99540ac1274a7ae9c77347ff2233257486cc72aa",
+      "scope": "Payer P2P exact reruns t016/t019/t036 with claude-sonnet-5"
+    }
+  ],
+  "stitched_resume": true,
+  "selection_policy": "Preserve the original Full75 packet except the maintainer-requested affected evidence. Correct cm_mdd_hard_refuses_002 using the valid v1.1.0 judge-only re-evaluation; retain the existing Payer packet trial files because t016/t019/t036 remained passes, with exact rerun evidence recorded in repair_evidence.json. Preserve all other 71 trial artifacts exactly.",
+  "repair_evidence_file": "repair_evidence.json"
+}

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/repair_evidence.json

@@ -0,0 +1,81 @@
+{
+  "schema": "chi-bench/leaderboard-repair-evidence/v1",
+  "repaired_at": "2026-10-07T14:14:19Z",
+  "maintainer_requested_scope": {
+    "care": [
+      "cm_mdd_hard_refuses_002"
+    ],
+    "payer_p2p": [
+      "pa_t016_t016_o001_p01_p2p_payer",
+      "pa_t019_t019_o001_p01_p2p_payer",
+      "pa_t036_t036_o002_p01_p2p_payer"
+    ],
+    "preserved_unaffected_trials": 71
+  },
+  "runtime": {
+    "chi_bench_git_sha": "c05b225bbb408f78a3043b7d40f022255bd38b19",
+    "version": "1.1.0",
+    "dataset_revision": "chi-bench-v1.0.0"
+  },
+  "care": {
+    "run_id": 37625832254,
+    "artifact_id": 11484057950,
+    "artifact_digest": "sha256:d41e6e9d58730ff3405c625a79177e30f5a38e77c929ba9d9f44d4951cbc58f7",
+    "mode": "judge_only_from_preserved_final_state",
+    "judge_model": "claude-opus-4-7",
+    "judge_votes": 1,
+    "binary_reward": 0,
+    "fractional_reward": 0,
+    "rubric_yes_count": 0,
+    "rubric_total": 5,
+    "judge_unavailable_reason": null,
+    "source_scorecard_sha256": "ba54f8a0a0feb97890cef9eaf16e2905bc72a3bbae584d72a3c1ca8e934e57c4",
+    "exported_state_sha256": "06cde2195cdfd283d5305552a848067061d06766677a1181c8fe699ed6c17996"
+  },
+  "payer_p2p": {
+    "run_id": 37628570479,
+    "artifact_id": 11487743870,
+    "artifact_digest": "sha256:84011caf4bb0466f64837d3f326c48ff88c6f2dbb1877ed34c279f7e19c31d01",
+    "historical_hg_agent_ref": "d0d7f72c23a6bf71b325e6cf373c40319ff4963a",
+    "agent": "hg-closure-codex",
+    "agent_model": "openai/gpt-5.6-luna",
+    "p2p_simulator_model": "claude-sonnet-5",
+    "judge_model": "claude-opus-4-7",
+    "judge_votes": 1,
+    "total_openai_agent_cost_usd": 0.16009574,
+    "tasks": [
+      {
+        "task": "pa_t016_t016_o001_p01_p2p_payer",
+        "binary_reward": 1,
+        "fractional_reward": 1,
+        "judge_check_count": 7,
+        "result_sha256": "1a38242206cdd1183ec294c944fad13e3f834047335e240f36549b423a110e07",
+        "scorecard_sha256": "be15c172814b246385261d843bbcaf0541530e2bb5f8c2cc4301db08a37b24f1"
+      },
+      {
+        "task": "pa_t019_t019_o001_p01_p2p_payer",
+        "binary_reward": 1,
+        "fractional_reward": 1,
+        "judge_check_count": 7,
+        "result_sha256": "7cc13cef7f4687ef126cf5acd1141a654f42eb3347950a08ae99f8b7484420b1",
+        "scorecard_sha256": "0694a0df6474fa2e7324778c4671375abca2235665b10da110f3b61eb0240809"
+      },
+      {
+        "task": "pa_t036_t036_o002_p01_p2p_payer",
+        "binary_reward": 1,
+        "fractional_reward": 1,
+        "judge_check_count": 7,
+        "result_sha256": "c9022d0a4b84550c8d30eec65e0cb1305bcc6f9a30d6a677c9fe2bc2ef63ec28",
+        "scorecard_sha256": "3caf95a75efb7253abbe0cbf854b652efc151eede8beb2afa092eead911d229b"
+      }
+    ],
+    "note": "All three historical Payer tasks remained passes. Existing packet trial files are preserved to avoid mixing new textual verifier outputs with old binary trajectories; exact rerun evidence is anchored by this run/artifact and hashes."
+  },
+  "repaired_score": {
+    "provider": "11/25",
+    "payer": "7/25",
+    "care": "2/25",
+    "overall": "20/75",
+    "overall_pct": 26.666666666666668
+  }
+}

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/results.csv

@@ -0,0 +1,5 @@
+benchmark,dataset_version,submission_id,team,agent,model,domain,pass_at_1,n_trials,n_tasks,mean_cost_usd,mean_walltime_s,submitted_at
+chi-bench,chi-bench-v1.0.0,hg-chi-full75-20261006,HG-Market-System,hg-closure-codex,"mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)",overall,0.26666666666666666,75,75,0.33176541946666666,349.44047122666666,2026-10-06T11:18:00Z
+chi-bench,chi-bench-v1.0.0,hg-chi-full75-20261006,HG-Market-System,hg-closure-codex,"mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)",pa_provider,0.44,25,25,0.101629252,202.70591964,2026-10-06T11:18:00Z
+chi-bench,chi-bench-v1.0.0,hg-chi-full75-20261006,HG-Market-System,hg-closure-codex,"mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)",pa_um,0.28,25,25,0.0869921824,281.2458558,2026-10-06T11:18:00Z
+chi-bench,chi-bench-v1.0.0,hg-chi-full75-20261006,HG-Market-System,hg-closure-codex,"mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)",cm,0.08,25,25,0.806674824,564.36963824,2026-10-06T11:18:00Z

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/sub.yaml

@@ -0,0 +1,24 @@
+schema: chi-bench/submission/v1
+submission:
+  id: hg-chi-full75-20261006
+  team: HG-Market-System
+  contact: https://github.com/hgreco85
+  agent: hg-closure-codex
+  model: "mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)"
+  notes: >-
+    Completed Full75 packet with the four maintainer-requested evidence repairs applied on CHI-Bench v1.1.0. Care cm_mdd_hard_refuses_002 was judge-only re-evaluated from the preserved final state and adjudicated 0/5 rubrics. Payer P2P t016, t019 and t036 were rerun with the historical Full75 HG agent snapshot and CHI_BENCH_P2P_SIMULATOR_MODEL=claude-sonnet-5; all three remained passes. The other 71 trial artifacts are preserved. Judge model claude-opus-4-7, one vote. Agent API cost only; simulator/judge costs not included.
+run:
+  environment: docker
+  n_attempts: 1
+  concurrency: 3
+  max_retries: 0
+  timeout_multiplier: 1.0
+dataset:
+  version: chi-bench-v1.0.0
+  domains:
+  - pa_provider
+  - pa_um
+  - cm
+paths:
+  data_root: data
+  output_root: reconstructed-from-runs-37390621096-and-37444743430

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/submission.json

@@ -0,0 +1,86 @@
+{
+  "schema": "chi-bench/submission/v1",
+  "submission": {
+    "id": "hg-chi-full75-20261006",
+    "team": "HG-Market-System",
+    "contact": "https://github.com/hgreco85",
+    "agent": "hg-closure-codex",
+    "model": "mixed: openai/gpt-5.6-luna (pa_provider, pa_um) + openai/gpt-5.4-mini (cm)",
+    "notes": "Completed public Full75 packet, then repaired only the four maintainer-requested affected evidence points on CHI-Bench v1.1.0: Care cm_mdd_hard_refuses_002 was judge-only re-evaluated from the byte-preserved saved final state and adjudicated 0/5 rubrics; Payer P2P t016, t019 and t036 were rerun with the historical Full75 HG agent snapshot and CHI_BENCH_P2P_SIMULATOR_MODEL=claude-sonnet-5 and all remained passes. The other 71 trial artifacts are preserved. Provider/UM agent model openai/gpt-5.6-luna; CM agent model openai/gpt-5.4-mini. Judge model claude-opus-4-7, one vote. Agent API cost only; judge/simulator costs not included.",
+    "submitted_at": "2026-10-06T11:18:00Z"
+  },
+  "dataset": {
+    "name": "chi-bench",
+    "version": "chi-bench-v1.0.0",
+    "domains": [
+      "pa_provider",
+      "pa_um",
+      "cm"
+    ]
+  },
+  "results": {
+    "overall": {
+      "pass_at_1": 0.26666666666666666,
+      "n_trials": 75,
+      "n_tasks": 75,
+      "mean_cost_usd": 0.33176541946666666,
+      "mean_walltime_s": 349.44047122666666
+    },
+    "per_domain": {
+      "pa_provider": {
+        "pass_at_1": 0.44,
+        "n_trials": 25,
+        "n_tasks": 25,
+        "mean_cost_usd": 0.101629252,
+        "mean_walltime_s": 202.70591964
+      },
+      "pa_um": {
+        "pass_at_1": 0.28,
+        "n_trials": 25,
+        "n_tasks": 25,
+        "mean_cost_usd": 0.0869921824,
+        "mean_walltime_s": 281.2458558
+      },
+      "cm": {
+        "pass_at_1": 0.08,
+        "n_trials": 25,
+        "n_tasks": 25,
+        "mean_cost_usd": 0.806674824,
+        "mean_walltime_s": 564.36963824
+      }
+    },
+    "mean_cost_usd": 0.33176541946666666,
+    "mean_walltime_s": 349.44047122666666
+  },
+  "provenance": {
+    "chi_bench_git_sha": "c05b225bbb408f78a3043b7d40f022255bd38b19",
+    "image_digest": null,
+    "judge_model": "claude-opus-4-7",
+    "judge_num_votes": 1,
+    "harness_version": "0.1.0",
+    "hg_source_runs": [
+      {
+        "run_id": 37390621096,
+        "head_sha": "d0d7f72c23a6bf71b325e6cf373c40319ff4963a",
+        "scope": "25 provider + 25 payer + first 15 verified care"
+      },
+      {
+        "run_id": 37444743430,
+        "head_sha": "226e47675520a4408a014edbf8483f64024007ba",
+        "scope": "exact remaining 10 care tasks"
+      },
+      {
+        "run_id": 37625832254,
+        "head_sha": "36e19b84b64ca59c30acf933de612a04e8f37a16",
+        "scope": "judge-only v1.1.0 repair of cm_mdd_hard_refuses_002 from preserved final state"
+      },
+      {
+        "run_id": 37628570479,
+        "head_sha": "99540ac1274a7ae9c77347ff2233257486cc72aa",
+        "scope": "exact v1.1.0 Payer P2P reruns t016/t019/t036 with claude-sonnet-5"
+      }
+    ],
+    "stitched_resume": true,
+    "selection_policy": "Preserve the original completed Full75 packet except for the maintainer-requested affected evidence. Integrate the valid judge-only Care re-evaluation for cm_mdd_hard_refuses_002, which changes that task from pass to fail. Preserve the three historical Payer trial packet files because all three tasks remained passes; attach exact v1.1.0 rerun evidence in repair_evidence.json. Preserve all other 71 trial artifacts exactly."
+  }
+}

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/trials/cm/cm_afib_moderate_anxious_001/result.json

@@ -0,0 +1,117 @@
+{
+    "id": "218f65b4-7f31-4487-946f-898d4ff6769e",
+    "task_name": "actava-ai/cm_afib_moderate_anxious_001",
+    "trial_name": "cm_afib_moderate_anxious_001__bRkwVnb",
+    "trial_uri": "file:///home/runner/work/HG-Market-System/HG-Market-System/chi-bench/trials/full75/care/cm_afib_moderate_anxious_001/cm_afib_moderate_anxious_001__bRkwVnb",
+    "task_id": {
+        "path": "/home/runner/work/HG-Market-System/HG-Market-System/chi-bench/data/care_management/tasks/cm_afib_moderate_anxious_001"
+    },
+    "source": null,
+    "task_checksum": "a43c14fadec72f11c1de9af6ee038fd27d3c453ce78dd9de56d454e395e33846",
+    "config": {
+        "task": {
+            "path": "/home/runner/work/HG-Market-System/HG-Market-System/chi-bench/data/care_management/tasks/cm_afib_moderate_anxious_001",
+            "git_url": null,
+            "git_commit_id": null,
+            "name": null,
+            "ref": null,
+            "overwrite": false,
+            "download_dir": null,
+            "source": null
+        },
+        "trial_name": "cm_afib_moderate_anxious_001__bRkwVnb",
+        "trials_dir": "/home/runner/work/HG-Market-System/HG-Market-System/chi-bench/trials/full75/care/cm_afib_moderate_anxious_001",
+        "install_only": false,
+        "timeout_multiplier": 1.0,
+        "agent_timeout_multiplier": null,
+        "verifier_timeout_multiplier": null,
+        "agent_setup_timeout_multiplier": null,
+        "environment_build_timeout_multiplier": null,
+        "agent": {
+            "name": null,
+            "import_path": "chi_bench.experiment.agents.hg_closure_codex_harness:HGClosureCodexHarness",
+            "model_name": "openai/gpt-5.4-mini",
+            "n_concurrent": null,
+            "concurrency_group": null,
+            "skills": [],
+            "override_timeout_sec": null,
+            "override_setup_timeout_sec": null,
+            "max_timeout_sec": null,
+            "resume_trajectory": false,
+            "load_trajectory": null,
+            "extra_allowed_hosts": [],
+            "kwargs": {},
+            "env": {
+                "OPENAI_API_KEY": "${OPENAI_API_KEY}",
+                "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}"
+            },
+            "mcp_servers": []
+        },
+        "environment": {
+            "type": null,
+            "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment",
+            "force_build": false,
+            "delete": true,
+            "cpu_enforcement_policy": "auto",
+            "memory_enforcement_policy": "auto",
+            "override_cpus": null,
+            "override_memory_mb": null,
+            "override_storage_mb": null,
+            "override_gpus": null,
+            "override_tpu": null,
+            "mounts": null,
+            "extra_docker_compose": [],
+            "kwargs": {},
+            "extra_allowed_hosts": []
+        },
+        "verifier": {
+            "override_timeout_sec": null,
+            "max_timeout_sec": null,
+            "disable": false
+        },
+        "artifacts": [],
+        "extra_instruction_paths": [],
+        "job_id": null
+    },
+    "agent_info": {
+        "name": "hg-closure-codex",
+        "version": "0.160.1",
+        "model_info": {
+            "name": "gpt-5.4-mini",
+            "provider": "openai"
+        }
+    },
+    "agent_result": {
+        "n_input_tokens": 3910228,
+        "n_cache_tokens": 3710464,
+        "n_output_tokens": 49178,
+        "cost_usd": 0.6494088,
+        "rollout_details": null,
+        "metadata": null
+    },
+    "verifier_result": {
+        "rewards": {
+            "reward": 0.0
+        }
+    },
+    "exception_info": null,
+    "started_at": "2026-10-06T01:01:19.594167Z",
+    "finished_at": "2026-10-06T01:11:48.421469Z",
+    "environment_setup": {
+        "started_at": "2026-10-06T01:01:19.741187Z",
+        "finished_at": "2026-10-06T01:01:56.313428Z"
+    },
+    "agent_setup": {
+        "started_at": "2026-10-06T01:01:56.313497Z",
+        "finished_at": "2026-10-06T01:02:29.431701Z"
+    },
+    "agent_execution": {
+        "started_at": "2026-10-06T01:02:29.431821Z",
+        "finished_at": "2026-10-06T01:09:01.090631Z"
+    },
+    "verifier": {
+        "started_at": "2026-10-06T01:09:07.573672Z",
+        "finished_at": "2026-10-06T01:11:47.702125Z"
+    },
+    "step_results": null
+}

benchmarks/chi-bench/submissions/2026-10-06-hg-chi-full75-20261006/trials/cm/cm_afib_moderate_anxious_001/verifier/reward.json

@@ -0,0 +1 @@
+{"reward": 0.0}