test(chunking): Capture semantic chunking benchmark evidence

.agents/skills/warden-qa/SKILL.md

@@ -0,0 +1,74 @@
+---
+name: warden-qa
+description: Concise local QA workflow for Warden changes. Use when asked to manually QA Warden, verify scanner behavior, exercise config or chunking changes, or sanity-check local output before finishing.
+---
+
+Use one targeted local run to prove the changed Warden behavior. Prefer the
+smallest deterministic path that exercises the real code.
+
+## Choose the Probe
+
+- For diff parsing, chunking, trigger matching, config loading, or report
+  shaping, use a focused `tsx` snippet against package APIs.
+- For CLI behavior, use `pnpm cli -- ...` with the narrowest command and flags
+  that reach the changed path.
+- For model-backed scanner behavior, use one small fixture and one skill. If
+  credentials or network access block the run, report that and verify the
+  deterministic pre-model path instead.
+- Do not run broad, expensive, or unrelated Warden scans for a narrow change.
+
+## Common Commands
+
+Run package API probes from `packages/warden` when they do not need a built CLI:
+
+```sh
+pnpm --filter @sentry/warden exec tsx -e '<targeted TypeScript probe>'
+```
+
+Run Warden through the repo wrapper when validating operator-facing CLI behavior:
+
+```sh
+pnpm cli -- <command> --json
+pnpm cli -- <command> --log
+```
+
+For chunking changes, verify the prepared file shape directly:
+
+```sh
+pnpm --filter @sentry/warden exec tsx -e '<prepareFiles probe printing files, chunks, contentMode, changedLineMap>'
+```
+
+Use synthetic patches when they prove the behavior more clearly than a live PR.
+Keep them tiny: one file, two or three hunks, and expected changed line ranges.
+
+## What to Inspect
+
+- The exact config that loaded.
+- File count and chunk count.
+- `ReviewChunk.contentMode`.
+- `ReviewChunk.changedLineMap`.
+- Any rendered prompt or JSON output that proves the changed path.
+- Exit status and the key output, not the full transcript.
+
+## Failure Handling
+
+If the output is too broad to prove the change, narrow the fixture or command.
+If a live scanner run is blocked by credentials, model gateway access, or
+network restrictions, say so and show the deterministic local evidence that was
+still checked.
+
+Do not hide uncertainty behind a passing test command. Name what local QA did
+not prove.
+
+## Reporting
+
+Report:
+
+- pass or fail
+- exact command run
+- key observed output
+- what behavior the output proves
+- anything still unproven locally
+
+Keep automated validation such as `pnpm lint`, `pnpm build`, `pnpm test`, and
+typechecks in a separate validation note. Manual QA is the local behavior probe.

.github/workflows/evals.yml

@@ -85,6 +85,8 @@ jobs:
     timeout-minutes: 90
     env:
       ANTHROPIC_API_KEY: ${{ secrets.WARDEN_ANTHROPIC_API_KEY }}
+      WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY || secrets.OPENROUTER_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY || secrets.WARDEN_OPENROUTER_API_KEY }}
       EVAL_SCORE_BASELINE: "0.75"
     steps:
       - uses: actions/checkout@v4
@@ -99,8 +101,8 @@ jobs:
 
       - name: Verify eval secret
         run: |
-          if [ -z "$ANTHROPIC_API_KEY" ]; then
-            echo "WARDEN_ANTHROPIC_API_KEY is required to run evals in CI" >&2
+          if [ -z "$OPENROUTER_API_KEY" ] && [ -z "$WARDEN_OPENROUTER_API_KEY" ]; then
+            echo "OPENROUTER_API_KEY or WARDEN_OPENROUTER_API_KEY is required to run evals in CI" >&2
             exit 1
           fi
 

.github/workflows/warden.yml

@@ -18,6 +18,8 @@ jobs:
       WARDEN_MODEL: anthropic/claude-sonnet-4-6
       WARDEN_OPENAI_API_KEY: ${{ secrets.WARDEN_OPENAI_API_KEY }}
       WARDEN_ANTHROPIC_API_KEY: ${{ secrets.WARDEN_ANTHROPIC_API_KEY }}
+      WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY || secrets.OPENROUTER_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY || secrets.WARDEN_OPENROUTER_API_KEY }}
       WARDEN_SENTRY_DSN: ${{ secrets.WARDEN_SENTRY_DSN }}
     steps:
       - uses: actions/checkout@v4

packages/evals/eval-bug-detection.yaml

@@ -1,6 +1,6 @@
 skill: skills/bug-detection.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openrouter/anthropic/claude-sonnet-4.6
 
 evals:
   - name: null-property-access

packages/evals/eval-precision.yaml

@@ -1,6 +1,6 @@
 skill: skills/precision.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openrouter/anthropic/claude-sonnet-4.6
 
 evals:
   - name: ignores-style-issues

packages/evals/eval-security-scanning.yaml

@@ -1,6 +1,6 @@
 skill: skills/security-scanning.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openrouter/anthropic/claude-sonnet-4.6
 
 evals:
   - name: sql-injection

packages/evals/src/auth.ts

@@ -0,0 +1,46 @@
+import { bridgeWardenProviderApiKeyEnv } from '../../warden/src/utils/index.js';
+export { DEFAULT_EVAL_MODEL, DEFAULT_EVAL_RUNTIME } from './types.js';
+import { DEFAULT_EVAL_MODEL } from './types.js';
+
+function providerFromModel(model: string): string | undefined {
+  const slashIndex = model.indexOf('/');
+  if (slashIndex <= 0) {
+    return undefined;
+  }
+
+  return model.slice(0, slashIndex);
+}
+
+function providerEnvPrefix(provider: string): string {
+  return provider.toUpperCase().replace(/-/g, '_');
+}
+
+/**
+ * Returns a provider API key from the env for eval skip checks.
+ */
+export function getEvalProviderApiKey(model = defaultEvalModel()): string {
+  bridgeWardenProviderApiKeyEnv();
+
+  const provider = providerFromModel(model);
+  if (!provider) {
+    return '';
+  }
+
+  const prefix = providerEnvPrefix(provider);
+  return process.env[`WARDEN_${prefix}_API_KEY`] ?? process.env[`${prefix}_API_KEY`] ?? '';
+}
+
+/**
+ * Returns the legacy runtime API key override only for direct Anthropic Pi models.
+ */
+export function getEvalRuntimeApiKey(model = defaultEvalModel()): string {
+  const provider = providerFromModel(model);
+  return provider === 'anthropic' ? getEvalProviderApiKey(model) : '';
+}
+
+/**
+ * Returns the eval model override or the repo's OpenRouter default.
+ */
+export function defaultEvalModel(): string {
+  return process.env['WARDEN_MODEL'] ?? DEFAULT_EVAL_MODEL;
+}

packages/evals/src/code-review.eval.ts

@@ -7,13 +7,16 @@ import {
 } from './harness.js';
 import { discoverEvalScenarios } from './index.js';
 import { formatEvalId, formatEvalTestName } from './names.js';
+import { DEFAULT_EVAL_RUNTIME, defaultEvalModel, getEvalProviderApiKey, getEvalRuntimeApiKey } from './auth.js';
 
-const apiKey = process.env['ANTHROPIC_API_KEY'] ?? '';
+const model = defaultEvalModel();
+const apiKey = getEvalRuntimeApiKey(model);
+const providerApiKey = getEvalProviderApiKey(model);
 const evals = discoverEvalScenarios({
   category: 'code-review',
   skill: '../warden/src/builtin-skills/code-review/SKILL.md',
-  runtime: 'pi',
-  model: 'anthropic/claude-sonnet-4-6',
+  runtime: DEFAULT_EVAL_RUNTIME,
+  model,
 });
 const CODE_REVIEW_RUN_TIMEOUT_MS = 120_000;
 const CODE_REVIEW_EVAL_TIMEOUT_MS = CODE_REVIEW_RUN_TIMEOUT_MS + 60_000;
@@ -30,7 +33,7 @@ describeEval(
     }),
     judges: [createWardenEvalJudge(apiKey)],
     judgeThreshold: 1,
-    skipIf: () => !apiKey,
+    skipIf: () => !providerApiKey,
   },
   (it) => {
     for (const meta of evals) {

packages/evals/src/e2e.eval.ts

@@ -7,20 +7,30 @@ import {
 } from './harness.js';
 import { discoverEvals } from './index.js';
 import { formatEvalId, formatEvalTestName } from './names.js';
+import {
+  DEFAULT_EVAL_RUNTIME,
+  defaultEvalModel,
+  getEvalProviderApiKey,
+  getEvalRuntimeApiKey,
+} from './auth.js';
 
-const apiKey = process.env['ANTHROPIC_API_KEY'] ?? '';
+const model = defaultEvalModel();
+const apiKey = getEvalRuntimeApiKey(model);
+const providerApiKey = getEvalProviderApiKey(model);
 const evals = discoverEvals();
 
 describeEval(
   'e2e',
   {
     harness: createWardenEvalHarness({
       apiKey,
+      runtime: DEFAULT_EVAL_RUNTIME,
+      model,
       verbose: true,
     }),
     judges: [createWardenEvalJudge(apiKey)],
     judgeThreshold: 1,
-    skipIf: () => !apiKey,
+    skipIf: () => !providerApiKey,
   },
   (it) => {
     for (const meta of evals) {

packages/evals/src/harness.ts

@@ -174,7 +174,13 @@ export function createWardenEvalJudge(apiKey: string) {
 
     const meta = input;
     const findings = output.data.findings;
-    const judgeResult = await runJudge(meta, findings, apiKey);
+    const judgeResult = await runJudge(meta, findings, {
+      apiKey,
+      runtime: output.data.runtime === 'claude' || output.data.runtime === 'pi'
+        ? output.data.runtime
+        : meta.runtime,
+      model: output.data.model ?? meta.model,
+    });
     if (judgeResult.error) {
       return {
         score: 0,

packages/evals/src/judge.ts

@@ -1,12 +1,16 @@
-import Anthropic from '@anthropic-ai/sdk';
-import { anthropicUsageToStats, parseJsonFromOutput, type Finding, type UsageStats } from '@sentry/warden';
+import { getRuntime, type Finding, type RuntimeName, type UsageStats } from '@sentry/warden';
 import type { EvalMeta, JudgeResponse } from './types.js';
-import { DEFAULT_EVAL_MODEL, JudgeResponseSchema } from './types.js';
+import { DEFAULT_EVAL_MODEL, DEFAULT_EVAL_RUNTIME, JudgeResponseSchema } from './types.js';
 
-const JUDGE_MODEL = DEFAULT_EVAL_MODEL;
 const JUDGE_MAX_TOKENS = 4096;
 const JUDGE_TIMEOUT_MS = 30_000;
 
+export interface RunJudgeOptions {
+  apiKey?: string;
+  runtime?: RuntimeName;
+  model?: string;
+}
+
 export interface JudgeResult {
   response: JudgeResponse;
   usage: UsageStats;
@@ -107,22 +111,24 @@ Requirements:
 export async function runJudge(
   meta: EvalMeta,
   findings: Finding[],
-  apiKey: string
+  options: RunJudgeOptions = {}
 ): Promise<JudgeResult> {
-  const client = new Anthropic({ apiKey, timeout: JUDGE_TIMEOUT_MS, maxRetries: 0 });
-
+  const runtimeName = options.runtime ?? DEFAULT_EVAL_RUNTIME;
+  const model = options.model ?? DEFAULT_EVAL_MODEL;
   const prompt = buildJudgePrompt(meta, findings);
-
-  const messages: Anthropic.MessageParam[] = [
-    { role: 'user', content: prompt },
-  ];
-
-  let response: Anthropic.Message;
+  const runtime = getRuntime(runtimeName);
+  let result: Awaited<ReturnType<typeof runtime.runAuxiliary<JudgeResponse>>>;
   try {
-    response = await client.messages.create({
-      model: JUDGE_MODEL,
-      max_tokens: JUDGE_MAX_TOKENS,
-      messages,
+    result = await runtime.runAuxiliary({
+      task: 'eval_judge',
+      agentName: 'warden-eval-judge',
+      apiKey: options.apiKey,
+      prompt,
+      schema: JudgeResponseSchema,
+      model,
+      maxTokens: JUDGE_MAX_TOKENS,
+      timeout: JUDGE_TIMEOUT_MS,
+      maxRetries: 0,
     });
   } catch (error) {
     const reason = `Judge API call failed: ${error instanceof Error ? error.message : String(error)}`;
@@ -144,58 +150,36 @@ export async function runJudge(
     };
   }
 
-  const usage = anthropicUsageToStats(JUDGE_MODEL, response.usage);
-
-  const textBlock = response.content.find(
-    (b): b is Anthropic.TextBlock => b.type === 'text'
-  );
-
-  if (!textBlock) {
-    return {
-      response: buildFallbackResponse(meta, 'No text in judge response'),
-      usage,
-      error: 'No text in judge response',
-    };
-  }
-
-  const parsed = await parseJsonFromOutput({
-    output: textBlock.text,
-    schema: JudgeResponseSchema,
-  });
-
-  if (!parsed.success) {
-    const reason = `Judge response parse failed: ${parsed.error}`;
+  if (!result.success) {
+    const reason = `Judge response failed: ${result.error}`;
     return {
       response: buildFallbackResponse(meta, reason),
-      usage,
+      usage: result.usage,
       error: reason,
     };
   }
 
   // Validate array lengths match assertions
-  const judgeResp = parsed.data;
+  const judgeResp = result.data;
   if (judgeResp.expectations.length !== meta.should_find.length) {
     return {
       response: buildFallbackResponse(meta, `Judge returned ${judgeResp.expectations.length} verdicts, expected ${meta.should_find.length}`),
-      usage,
+      usage: result.usage,
       error: `Judge returned ${judgeResp.expectations.length} verdicts, expected ${meta.should_find.length}`,
     };
   }
   if (judgeResp.antiExpectations.length !== meta.should_not_find.length) {
     return {
       response: buildFallbackResponse(meta, `Judge returned ${judgeResp.antiExpectations.length} anti-verdicts, expected ${meta.should_not_find.length}`),
-      usage,
+      usage: result.usage,
       error: `Judge returned ${judgeResp.antiExpectations.length} anti-verdicts, expected ${meta.should_not_find.length}`,
     };
   }
 
-  return { response: judgeResp, usage };
+  return { response: judgeResp, usage: result.usage };
 }
 
-/**
- * Build a fallback judge response when parsing fails.
- * Marks all assertions as not met with the error reason.
- */
+/** Build a failed judge response when the judge cannot produce a usable verdict. */
 function buildFallbackResponse(meta: EvalMeta, reason: string): JudgeResponse {
   return {
     expectations: meta.should_find.map(() => ({