eval(workbench): web-design-guidelines eval suite + proposed upstream changes

examples/workbench/web-design-guidelines/README.md

@@ -0,0 +1,81 @@
+# web-design-guidelines eval
+
+Eval suite for [`vercel-labs/agent-skills/web-design-guidelines`](https://github.com/vercel-labs/agent-skills) — a skill that reviews UI files for Vercel Web Interface Guidelines compliance.
+
+## Cases
+
+Each case runs the skill on a focused TSX sample with seeded violations and
+grades whether the agent's findings cover them. We split coverage across files
+to mirror real usage (a developer reviews one file at a time, not a kitchen
+sink) and to avoid overwhelming smaller models.
+
+### `review-product-card` — Accessibility + Focus States
+
+Sample: `workspace/ProductCard.tsx`
+
+| Line | Violation | Rule |
+|---|---|---|
+| 15 | `<img>` without `alt` | Images need `alt` (or `alt=""` if decorative) |
+| 18 | `<div onClick>` for action | `<button>` for actions, not `<div onClick>` |
+| 21–23 | Icon-only `<button>` without `aria-label` | Icon-only buttons need `aria-label` |
+| 24–29 | `<input>` without `<label>` or `aria-label` | Form controls need `<label>` or `aria-label` |
+| 30–32 | `outline-none` className without focus replacement | Never `outline-none` without focus replacement |
+
+### `review-checkout-form` — Forms
+
+Sample: `workspace/CheckoutForm.tsx`
+
+| Line | Violation | Rule |
+|---|---|---|
+| 17 | `<label>` without `htmlFor` | Labels clickable (`htmlFor` or wrapping control) |
+| 18–25 | `<input>` for email uses `type="text"` | Use correct `type` (`email`, `tel`, `url`, `number`) |
+| 18–25 | `<input>` missing `autoComplete` | Inputs need `autocomplete` and meaningful `name` |
+| 24 | `onPaste={(e) => e.preventDefault()}` | Never block paste |
+| 30 | Submit button `disabled` before request starts | Submit stays enabled until request starts |
+
+### `review-loading-screen` — Typography + Content Handling
+
+Sample: `workspace/LoadingScreen.tsx`
+
+| Line | Violation | Rule |
+|---|---|---|
+| 12 | `"Loading..."` (three dots, not `…`) | `…` not `...`; loading states end with `…` |
+| 13 | Straight quotes `"..."` | Curly quotes `"..."` not straight |
+| 14 | `{fileSize} MB` without `&nbsp;` | Non-breaking spaces between number and unit |
+| 15–18 | Flex children without `min-w-0` for `truncate` | Flex children need `min-w-0` |
+| 19–23 | `recentFiles.map(...)` no empty-state branch | Handle empty states |
+
+### `review-hero-section` — Animation + Images + Performance
+
+Sample: `workspace/HeroSection.tsx`
+
+| Line | Violation | Rule |
+|---|---|---|
+| 6 | Above-fold `<img>` missing `width`/`height` | `<img>` needs explicit `width` and `height` (CLS) |
+| 6 | Above-fold `<img>` missing `priority`/`fetchpriority="high"` | Above-fold critical images need priority hint |
+| 7–10 | `transition: 'all'` | Never `transition: all` — list properties explicitly |
+| 15–18 | Animation without `prefers-reduced-motion` consideration | Honor `prefers-reduced-motion` |
+| 23 | Below-fold `<img>` missing `loading="lazy"` | Below-fold images need `loading="lazy"` |
+
+## Vendored snapshot
+
+The skill normally `WebFetch`es its rules from `vercel-labs/web-interface-guidelines`. For deterministic eval, we vendor a snapshot at `references/web-design-guidelines/command.md` and tweak `SKILL.md` to read the local copy. The diff vs upstream is one section (`Guidelines Source` → local file).
+
+## Run
+
+```bash
+export OPENROUTER_API_KEY=sk-or-...
+npx tsx src/cli.ts run-suite examples/workbench/web-design-guidelines/suite.yml --trials 3
+```
+
+## Models
+
+The suite is set up for a 3-provider mid-tier matrix:
+
+- `openrouter/anthropic/claude-sonnet-4.6`
+- `openrouter/openai/gpt-5-mini`
+- `openrouter/google/gemini-2.5-pro`
+
+## Graders
+
+One grader per case under `checks/`. Each reads `/work/findings.txt`, extracts every `<file>.tsx:<line>` reference, and confirms each expected violation is identified by both line number (within an accepted range for multi-line elements) and a keyword match. `pass` requires all expected violations; `score` is the fraction found.

examples/workbench/web-design-guidelines/checks/_grader-utils.mjs

@@ -0,0 +1,70 @@
+// Shared grader logic for web-design-guidelines eval cases.
+//
+// Each finding is assumed to be one line in findings.txt that references
+// "<File>.tsx:<line>" (line numbers come from the agent — they're often
+// off by ±1-2 due to LLM line-counting). A violation is considered "found"
+// when at least one finding line:
+//   (a) references a line number within the violation's accepted range, AND
+//   (b) contains at least one of the violation's distinguishing keywords.
+//
+// This per-finding-line check prevents spurious cross-matches (e.g. the
+// keyword "label" from a different finding being credited to a paste rule).
+
+import { existsSync, readFileSync } from 'node:fs';
+
+export function gradeFindings({ findingsPath, file, expected }) {
+  const failures = [];
+  const found = new Set();
+
+  if (!existsSync(findingsPath)) {
+    failures.push('findings.txt was not created');
+    return emitResult({ found, expected, failures });
+  }
+
+  const text = readFileSync(findingsPath, 'utf-8');
+  const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i');
+  const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln));
+
+  for (const v of expected) {
+    for (const line of findingLines) {
+      const m = line.match(refRe);
+      if (!m) continue;
+      const lineNum = Number(m[1]);
+      if (!v.lines.includes(lineNum)) continue;
+      if (!v.keywords.some((re) => re.test(line))) continue;
+      found.add(v.id);
+      break;
+    }
+  }
+
+  return emitResult({ found, expected, failures });
+}
+
+function emitResult({ found, expected, failures }) {
+  const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id);
+  const score = found.size / expected.length;
+  const pass = found.size === expected.length;
+
+  console.log(JSON.stringify({
+    pass,
+    score,
+    evidence: [
+      `${found.size}/${expected.length} expected violations identified`,
+      ...[...found].map((id) => `+ ${id}`),
+      ...missing.map((id) => `- missing: ${id}`),
+      ...failures,
+    ],
+  }));
+  return pass;
+}
+
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+// Helper: build an inclusive line range [start, start+1, ..., end].
+export function range(start, end) {
+  const out = [];
+  for (let i = start; i <= end; i++) out.push(i);
+  return out;
+}

examples/workbench/web-design-guidelines/checks/grade-a11y-findings.mjs

@@ -0,0 +1,37 @@
+import { join } from 'node:path';
+import { gradeFindings, range } from './_grader-utils.mjs';
+
+const expected = [
+  {
+    id: 'img-missing-alt',
+    lines: range(13, 17),
+    keywords: [/\balt\b/i, /\bimg\b/i],
+  },
+  {
+    id: 'div-onclick-instead-of-button',
+    lines: range(16, 22),
+    keywords: [/\bdiv\b.*\bonclick\b|\bonclick\b.*\bdiv\b/i, /<button>|\bbutton\b/i],
+  },
+  {
+    id: 'icon-only-button-no-aria-label',
+    lines: range(19, 25),
+    keywords: [/aria-label/i, /icon-only/i],
+  },
+  {
+    id: 'input-without-label',
+    lines: range(22, 31),
+    keywords: [/\blabel\b/i, /aria-label/i],
+  },
+  {
+    id: 'outline-none-no-focus',
+    lines: range(28, 34),
+    keywords: [/outline-?none/i, /\bfocus\b/i],
+  },
+];
+
+const pass = gradeFindings({
+  findingsPath: join(process.env.WORK, 'findings.txt'),
+  file: 'ProductCard.tsx',
+  expected,
+});
+process.exit(pass ? 0 : 1);

examples/workbench/web-design-guidelines/checks/grade-animation-findings.mjs

@@ -0,0 +1,37 @@
+import { join } from 'node:path';
+import { gradeFindings, range } from './_grader-utils.mjs';
+
+const expected = [
+  {
+    id: 'img-missing-width-height',
+    lines: range(4, 8),
+    keywords: [/\bwidth\b/i, /\bheight\b/i, /\bcls\b|layout\s*shift/i],
+  },
+  {
+    id: 'above-fold-img-missing-priority',
+    lines: range(4, 8),
+    keywords: [/\bpriority\b/i, /fetchpriority/i],
+  },
+  {
+    id: 'transition-all',
+    lines: range(5, 12),
+    keywords: [/transition.*all|transition:\s*['"]?all/i],
+  },
+  {
+    id: 'animation-no-prefers-reduced-motion',
+    lines: range(13, 20),
+    keywords: [/prefers-?reduced-?motion|reduce[d-]?motion/i],
+  },
+  {
+    id: 'below-fold-img-missing-lazy',
+    lines: range(21, 25),
+    keywords: [/\blazy\b|loading=['"]?lazy/i],
+  },
+];
+
+const pass = gradeFindings({
+  findingsPath: join(process.env.WORK, 'findings.txt'),
+  file: 'HeroSection.tsx',
+  expected,
+});
+process.exit(pass ? 0 : 1);

examples/workbench/web-design-guidelines/checks/grade-form-findings.mjs

@@ -0,0 +1,37 @@
+import { join } from 'node:path';
+import { gradeFindings, range } from './_grader-utils.mjs';
+
+const expected = [
+  {
+    id: 'label-without-htmlfor',
+    lines: range(15, 19),
+    keywords: [/\blabel\b/i, /\bhtml-?for\b|wrapping|clickable/i],
+  },
+  {
+    id: 'wrong-input-type-for-email',
+    lines: range(16, 27),
+    keywords: [/\btype\b/i, /\bemail\b/i],
+  },
+  {
+    id: 'input-missing-autocomplete',
+    lines: range(16, 27),
+    keywords: [/auto-?complete/i],
+  },
+  {
+    id: 'block-paste',
+    lines: range(21, 26),
+    keywords: [/\bpaste\b/i],
+  },
+  {
+    id: 'submit-button-disabled-pre-request',
+    lines: range(28, 32),
+    keywords: [/\bdisabled\b/i, /\bsubmit\b|\bbutton\b/i],
+  },
+];
+
+const pass = gradeFindings({
+  findingsPath: join(process.env.WORK, 'findings.txt'),
+  file: 'CheckoutForm.tsx',
+  expected,
+});
+process.exit(pass ? 0 : 1);

examples/workbench/web-design-guidelines/checks/grade-typography-findings.mjs

@@ -0,0 +1,37 @@
+import { join } from 'node:path';
+import { gradeFindings, range } from './_grader-utils.mjs';
+
+const expected = [
+  {
+    id: 'three-dots-not-ellipsis',
+    lines: range(10, 14),
+    keywords: [/\.\.\./, /ellipsis/i, /…/, /loading\s*$|loading\s*[…\.]/i],
+  },
+  {
+    id: 'straight-quotes-not-curly',
+    lines: range(11, 15),
+    keywords: [/\bquot/i, /curly|smart\s*quotes|typographic/i],
+  },
+  {
+    id: 'missing-nbsp-between-number-and-unit',
+    lines: range(12, 16),
+    keywords: [/non-?breaking|nbsp| /i],
+  },
+  {
+    id: 'flex-child-no-min-w-0',
+    lines: range(13, 20),
+    keywords: [/min-w-0|min-?width/i],
+  },
+  {
+    id: 'no-empty-state-handling',
+    lines: range(15, 25),
+    keywords: [/empty[-\s]+state|empty\s+array|empty\s+list|empty\s*<ul>|unguarded|fallback/i],
+  },
+];
+
+const pass = gradeFindings({
+  findingsPath: join(process.env.WORK, 'findings.txt'),
+  file: 'LoadingScreen.tsx',
+  expected,
+});
+process.exit(pass ? 0 : 1);