eval(auto-pilot): batch 2 — 10 skills, 8/10 success, 0 failures

examples/workbench/building-native-ui/README.md

@@ -0,0 +1,54 @@
+# building-native-ui eval
+
+Eval suite for
+[`expo/skills/building-native-ui`](https://github.com/expo/skills) —
+complete guide for building beautiful apps with Expo Router, covering
+library preferences, styling, navigation, responsiveness, and behavior rules.
+
+## Cases
+
+### `review-media-player` — Library preferences, responsiveness, and styling
+
+Sample: `workspace/MediaPlayerScreen.tsx`
+
+| Line | Violation | Rule |
+|------|-----------|------|
+| 3    | `import { Video } from 'expo-av'` — should use `expo-video` | Library Preferences: `expo-video` not `expo-av` |
+| 5    | `Dimensions.get('window')` — should use `useWindowDimensions` | Responsiveness: prefer `useWindowDimensions` over `Dimensions.get()` |
+| 8    | `Platform.OS` — should use `process.env.EXPO_OS` | Library Preferences: `process.env.EXPO_OS` not `Platform.OS` |
+| 11   | `<SafeAreaView` from `react-native` — should use `react-native-safe-area-context` | Library Preferences: `react-native-safe-area-context` not RN SafeAreaView |
+| 32   | `shadowColor`, `shadowOffset`, `elevation` legacy shadow props | Styling/Shadows: use CSS `boxShadow`, NEVER legacy RN shadow/elevation |
+
+### `review-settings-screen` — Library preferences, responsiveness, and behavior
+
+Sample: `workspace/SettingsScreen.tsx`
+
+| Line | Violation | Rule |
+|------|-----------|------|
+| 2    | `Picker` imported from `react-native` — removed module | Library Preferences: never use modules removed from React Native |
+| 3    | `import Permissions from 'expo-permissions'` — deprecated | Library Preferences: never use legacy expo-permissions |
+| 8    | `useContext(ThemeContext)` — should use `React.use(ThemeContext)` | Library Preferences: `React.use` not `React.useContext` |
+| 11   | `<ScrollView` missing `contentInsetAdjustmentBehavior="automatic"` | Responsiveness: always use `contentInsetAdjustmentBehavior="automatic"` on ScrollView |
+| 14   | `<img` intrinsic element — should use `expo-image` Image | Library Preferences: `expo-image` Image instead of `img`; Behavior: never use intrinsic elements |
+
+## Vendored snapshot
+
+The skill normally ships as part of the `expo/skills` repo at
+`plugins/expo/skills/building-native-ui/SKILL.md`. For deterministic eval
+we vendor a snapshot at `references/building-native-ui/SKILL.md`. The skill
+has no remote fetch calls, so the diff vs upstream is zero (content identical).
+
+## Run
+
+```bash
+export OPENROUTER_API_KEY=sk-or-...
+npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3
+```
+
+## Models
+
+The suite runs a 3-provider mid-tier matrix:
+
+- `openrouter/anthropic/claude-sonnet-4-6`
+- `openrouter/openai/gpt-5-mini`
+- `openrouter/google/gemini-2.5-pro`

examples/workbench/building-native-ui/analysis.md

@@ -0,0 +1,18 @@
+---
+skill: expo/skills/building-native-ui
+status: success
+classification: code-patterns
+baseline_rule_coverage: 0.99
+final_rule_coverage: 0.99
+modifications_tried: 0
+total_cost_usd: 0.67
+---
+
+# Auto-pilot run for `expo/skills/building-native-ui`
+
+- Skill fetched from `https://github.com/expo/skills` — actual path is `plugins/expo/skills/building-native-ui/SKILL.md` (not the standard `skills/<id>/SKILL.md` layout)
+- Classified as **code-patterns**: comprehensive Expo Router UI guidelines covering library preferences, styling, navigation, responsiveness, and behavior rules; evaluated using code-reviewer shape (seed violations → grade findings)
+- Two sample files seeded with 5 violations each: `MediaPlayerScreen.tsx` (expo-av, Dimensions.get, Platform.OS, RN SafeAreaView, legacy shadow props) and `SettingsScreen.tsx` (expo-permissions, removed Picker, useContext→React.use, missing contentInsetAdjustmentBehavior, img element)
+- Baseline run: 3 trials × 2 cases × 3 models = 18 trials; 17/18 passed; 89/90 violations found across all graders (rule-coverage = 0.989)
+- Only miss: `useContext→React.use` rule missed once by gpt-5-mini (1/18 trials) — absence-of-alternative-API pattern per lessons.md, expected medium-high difficulty; single miss in 18 trials does not warrant modification
+- Exiting success per Phase 3 rule (baseline ≥ 0.95); no upstream skill changes proposed

examples/workbench/building-native-ui/checks/_grader-utils.mjs

@@ -0,0 +1,94 @@
+// Shared grader logic for web-design-guidelines eval cases.
+//
+// Each finding is assumed to be one line in findings.txt that references
+// "<File>.tsx:<line>" (line numbers come from the agent — they're often
+// off by ±1-2 due to LLM line-counting). A violation is considered "found"
+// when at least one finding line:
+//   (a) references a line number within the violation's accepted range, AND
+//   (b) contains at least one of the violation's distinguishing keywords.
+//
+// This per-finding-line check prevents spurious cross-matches (e.g. the
+// keyword "label" from a different finding being credited to a paste rule).
+
+import { existsSync, readFileSync } from 'node:fs';
+
+export function gradeFindings({ findingsPath, file, expected }) {
+  const failures = [];
+  const found = new Set();
+
+  if (!existsSync(findingsPath)) {
+    failures.push('findings.txt was not created');
+    return emitResult({ found, expected, failures });
+  }
+
+  const text = readFileSync(findingsPath, 'utf-8');
+  const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i');
+  const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln));
+
+  for (const v of expected) {
+    for (const line of findingLines) {
+      const m = line.match(refRe);
+      if (!m) continue;
+      const lineNum = Number(m[1]);
+      if (!v.lines.includes(lineNum)) continue;
+      if (!v.keywords.some((re) => re.test(line))) continue;
+      found.add(v.id);
+      break;
+    }
+  }
+
+  return emitResult({ found, expected, failures });
+}
+
+function emitResult({ found, expected, failures }) {
+  const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id);
+  const score = found.size / expected.length;
+  const pass = found.size === expected.length;
+
+  console.log(JSON.stringify({
+    pass,
+    score,
+    evidence: [
+      `${found.size}/${expected.length} expected violations identified`,
+      ...[...found].map((id) => `+ ${id}`),
+      ...missing.map((id) => `- missing: ${id}`),
+      ...failures,
+    ],
+  }));
+  return pass;
+}
+
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+// Helper: build an inclusive line range [start, start+1, ..., end].
+export function range(start, end) {
+  const out = [];
+  for (let i = start; i <= end; i++) out.push(i);
+  return out;
+}
+
+// Helper: centered loose range — accepts the violation line ± tolerance.
+// Default tolerance ±8 handles LLM line-counting drift on multi-line elements.
+// PREFER this over `range(N-3, N+3)` — see lessons.md § G1.
+export function looseRange(centerLine, tolerance = 8) {
+  return range(centerLine - tolerance, centerLine + tolerance);
+}
+
+// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')`
+// matches both "empty state" and "empty-state" and "emptystate".
+// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2.
+export function fuzzyKeyword(phrase) {
+  const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const flexible = escaped.replace(/\s+/g, '[-\\s]*');
+  return new RegExp(flexible, 'i');
+}
+
+// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches
+// "cover", "covering", "covered", "does not cover".
+// PREFER this over `/covering/i` — see lessons.md § G4.
+export function tolerantKeyword(stem) {
+  const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp(`\\b${escaped}\\w*`, 'i');
+}

examples/workbench/building-native-ui/checks/grade-media-player-findings.mjs

@@ -0,0 +1,44 @@
+// Grader for MediaPlayerScreen.tsx violations.
+// Expected: 5 violations seeded across library preferences, responsiveness, and styling rules.
+
+import { join } from 'node:path';
+import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs';
+
+const work = process.env.WORK;
+const findingsPath = join(work, 'findings.txt');
+const file = 'MediaPlayerScreen.tsx';
+
+const expected = [
+  {
+    // Line 3: `import { Video } from 'expo-av'` — should use expo-video, not expo-av
+    id: 'expo-av-not-expo-video',
+    lines: looseRange(3),
+    keywords: [fuzzyKeyword('expo-av'), tolerantKeyword('expo-video'), tolerantKeyword('expo-audio')],
+  },
+  {
+    // Line 5: `Dimensions.get('window')` — should use useWindowDimensions
+    id: 'dimensions-get-not-hook',
+    lines: looseRange(5),
+    keywords: [fuzzyKeyword('Dimensions'), fuzzyKeyword('useWindowDimensions'), fuzzyKeyword('window dimensions')],
+  },
+  {
+    // Line 8: `Platform.OS` — should use `process.env.EXPO_OS`
+    id: 'platform-os-not-expo-os',
+    lines: looseRange(8),
+    keywords: [fuzzyKeyword('Platform.OS'), fuzzyKeyword('EXPO_OS'), fuzzyKeyword('process.env')],
+  },
+  {
+    // Line 11: `<SafeAreaView` from react-native — should use react-native-safe-area-context
+    id: 'rn-safe-area-view-wrong-import',
+    lines: looseRange(11),
+    keywords: [fuzzyKeyword('SafeAreaView'), fuzzyKeyword('safe-area-context'), fuzzyKeyword('safe area context')],
+  },
+  {
+    // Line 32: `shadowColor` legacy shadow props — should use CSS `boxShadow`
+    id: 'legacy-shadow-not-box-shadow',
+    lines: looseRange(32),
+    keywords: [fuzzyKeyword('shadowColor'), fuzzyKeyword('boxShadow'), fuzzyKeyword('box shadow'), fuzzyKeyword('legacy shadow'), tolerantKeyword('elevation')],
+  },
+];
+
+gradeFindings({ findingsPath, file, expected });

examples/workbench/building-native-ui/checks/grade-settings-screen-findings.mjs

@@ -0,0 +1,44 @@
+// Grader for SettingsScreen.tsx violations.
+// Expected: 5 violations seeded across library preferences, responsiveness, and behavior rules.
+
+import { join } from 'node:path';
+import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs';
+
+const work = process.env.WORK;
+const findingsPath = join(work, 'findings.txt');
+const file = 'SettingsScreen.tsx';
+
+const expected = [
+  {
+    // Line 3: `import Permissions from 'expo-permissions'` — legacy, never use
+    id: 'expo-permissions-deprecated',
+    lines: looseRange(3),
+    keywords: [fuzzyKeyword('expo-permissions'), tolerantKeyword('permission'), fuzzyKeyword('deprecated')],
+  },
+  {
+    // Line 2: `Picker` imported from react-native — removed module, never use
+    id: 'picker-removed-from-rn',
+    lines: looseRange(2),
+    keywords: [tolerantKeyword('Picker'), fuzzyKeyword('removed'), fuzzyKeyword('react-native')],
+  },
+  {
+    // Line 8: `useContext(ThemeContext)` — should use `React.use(ThemeContext)`
+    id: 'use-context-not-react-use',
+    lines: looseRange(8),
+    keywords: [fuzzyKeyword('useContext'), fuzzyKeyword('React.use'), fuzzyKeyword('use context')],
+  },
+  {
+    // Line 11: `<ScrollView` missing `contentInsetAdjustmentBehavior="automatic"`
+    id: 'scroll-view-missing-content-inset',
+    lines: looseRange(11),
+    keywords: [fuzzyKeyword('contentInsetAdjustmentBehavior'), fuzzyKeyword('content inset'), fuzzyKeyword('automatic'), fuzzyKeyword('ScrollView')],
+  },
+  {
+    // Line 14: `<img` element — should use expo-image Image component
+    id: 'img-element-not-expo-image',
+    lines: looseRange(14),
+    keywords: [fuzzyKeyword('img'), fuzzyKeyword('expo-image'), fuzzyKeyword('Image'), tolerantKeyword('intrinsic')],
+  },
+];
+
+gradeFindings({ findingsPath, file, expected });