diff --git a/examples/workbench/building-native-ui/README.md b/examples/workbench/building-native-ui/README.md new file mode 100644 index 0000000..6d540f5 --- /dev/null +++ b/examples/workbench/building-native-ui/README.md @@ -0,0 +1,54 @@ +# building-native-ui eval + +Eval suite for +[`expo/skills/building-native-ui`](https://github.com/expo/skills) — +complete guide for building beautiful apps with Expo Router, covering +library preferences, styling, navigation, responsiveness, and behavior rules. + +## Cases + +### `review-media-player` — Library preferences, responsiveness, and styling + +Sample: `workspace/MediaPlayerScreen.tsx` + +| Line | Violation | Rule | +|------|-----------|------| +| 3 | `import { Video } from 'expo-av'` — should use `expo-video` | Library Preferences: `expo-video` not `expo-av` | +| 5 | `Dimensions.get('window')` — should use `useWindowDimensions` | Responsiveness: prefer `useWindowDimensions` over `Dimensions.get()` | +| 8 | `Platform.OS` — should use `process.env.EXPO_OS` | Library Preferences: `process.env.EXPO_OS` not `Platform.OS` | +| 11 | `/SKILL.md` layout) +- Classified as **code-patterns**: comprehensive Expo Router UI guidelines covering library preferences, styling, navigation, responsiveness, and behavior rules; evaluated using code-reviewer shape (seed violations → grade findings) +- Two sample files seeded with 5 violations each: `MediaPlayerScreen.tsx` (expo-av, Dimensions.get, Platform.OS, RN SafeAreaView, legacy shadow props) and `SettingsScreen.tsx` (expo-permissions, removed Picker, useContext→React.use, missing contentInsetAdjustmentBehavior, img element) +- Baseline run: 3 trials × 2 cases × 3 models = 18 trials; 17/18 passed; 89/90 violations found across all graders (rule-coverage = 0.989) +- Only miss: `useContext→React.use` rule missed once by gpt-5-mini (1/18 trials) — absence-of-alternative-API pattern per lessons.md, expected medium-high difficulty; single miss in 18 trials does not warrant modification +- Exiting success per Phase 3 rule (baseline ≥ 0.95); no upstream skill changes proposed diff --git a/examples/workbench/building-native-ui/checks/_grader-utils.mjs b/examples/workbench/building-native-ui/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/building-native-ui/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/building-native-ui/checks/grade-media-player-findings.mjs b/examples/workbench/building-native-ui/checks/grade-media-player-findings.mjs new file mode 100644 index 0000000..d5aa8fe --- /dev/null +++ b/examples/workbench/building-native-ui/checks/grade-media-player-findings.mjs @@ -0,0 +1,44 @@ +// Grader for MediaPlayerScreen.tsx violations. +// Expected: 5 violations seeded across library preferences, responsiveness, and styling rules. + +import { join } from 'node:path'; +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const work = process.env.WORK; +const findingsPath = join(work, 'findings.txt'); +const file = 'MediaPlayerScreen.tsx'; + +const expected = [ + { + // Line 3: `import { Video } from 'expo-av'` — should use expo-video, not expo-av + id: 'expo-av-not-expo-video', + lines: looseRange(3), + keywords: [fuzzyKeyword('expo-av'), tolerantKeyword('expo-video'), tolerantKeyword('expo-audio')], + }, + { + // Line 5: `Dimensions.get('window')` — should use useWindowDimensions + id: 'dimensions-get-not-hook', + lines: looseRange(5), + keywords: [fuzzyKeyword('Dimensions'), fuzzyKeyword('useWindowDimensions'), fuzzyKeyword('window dimensions')], + }, + { + // Line 8: `Platform.OS` — should use `process.env.EXPO_OS` + id: 'platform-os-not-expo-os', + lines: looseRange(8), + keywords: [fuzzyKeyword('Platform.OS'), fuzzyKeyword('EXPO_OS'), fuzzyKeyword('process.env')], + }, + { + // Line 11: `` instead of `` for smarter safe area insets +- `contentInsetAdjustmentBehavior="automatic"` should be applied to FlatList and SectionList as well +- Use flexbox instead of Dimensions API +- ALWAYS prefer `useWindowDimensions` over `Dimensions.get()` to measure screen size + +## Behavior + +- Use expo-haptics conditionally on iOS to make more delightful experiences +- Use views with built-in haptics like `` from React Native and `@react-native-community/datetimepicker` +- When a route belongs to a Stack, its first child should almost always be a ScrollView with `contentInsetAdjustmentBehavior="automatic"` set +- When adding a `ScrollView` to the page it should almost always be the first component inside the route component +- Prefer `headerSearchBarOptions` in Stack.Screen options to add a search bar +- Use the `` prop on text containing data that could be copied +- Consider formatting large numbers like 1.4M or 38k +- Never use intrinsic elements like 'img' or 'div' unless in a webview or Expo DOM component + +# Styling + +Follow Apple Human Interface Guidelines. + +## General Styling Rules + +- Prefer flex gap over margin and padding styles +- Prefer padding over margin where possible +- Always account for safe area, either with stack headers, tabs, or ScrollView/FlatList `contentInsetAdjustmentBehavior="automatic"` +- Ensure both top and bottom safe area insets are accounted for +- Inline styles not StyleSheet.create unless reusing styles is faster +- Add entering and exiting animations for state changes +- Use `{ borderCurve: 'continuous' }` for rounded corners unless creating a capsule shape +- ALWAYS use a navigation stack title instead of a custom text element on the page +- When padding a ScrollView, use `contentContainerStyle` padding and gap instead of padding on the ScrollView itself (reduces clipping) +- CSS and Tailwind are not supported - use inline styles + +## Text Styling + +- Add the `selectable` prop to every `` element displaying important data or error messages +- Counters should use `{ fontVariant: 'tabular-nums' }` for alignment + +## Shadows + +Use CSS `boxShadow` style prop. NEVER use legacy React Native shadow or elevation styles. + +```tsx + +``` + +'inset' shadows are supported. + +# Navigation + +## Link + +Use `` from 'expo-router' for navigation between routes. + +```tsx +import { Link } from 'expo-router'; + +// Basic link + + +// Wrapping custom components + + ... + +``` + +Whenever possible, include a `` to follow iOS conventions. Add context menus and previews frequently to enhance navigation. + +## Stack + +- ALWAYS use `_layout.tsx` files to define stacks +- Use Stack from 'expo-router/stack' for native navigation stacks + +### Page Title + +Set the page title in Stack.Screen options: + +```tsx + +``` + +## Context Menus + +Add long press context menus to Link components: + +```tsx +import { Link } from "expo-router"; + + + + + + + + + + + + {}} /> + {}} + /> + + +; +``` + +## Link Previews + +Use link previews frequently to enhance navigation: + +```tsx + + + + + + + + +``` + +Link preview can be used with context menus. + +## Modal + +Present a screen as a modal: + +```tsx + +``` + +Prefer this to building a custom modal component. + +## Sheet + +Present a screen as a dynamic form sheet: + +```tsx + +``` + +- Using `contentStyle: { backgroundColor: "transparent" }` makes the background liquid glass on iOS 26+. + +## Common route structure + +A standard app layout with tabs and stacks inside each tab: + +``` +app/ + _layout.tsx — + (index,search)/ + _layout.tsx — + index.tsx — Main list + search.tsx — Search view +``` + +```tsx +// app/_layout.tsx +import { NativeTabs, Icon, Label } from "expo-router/unstable-native-tabs"; +import { Theme } from "../components/theme"; + +export default function Layout() { + return ( + + + + + + + + + + ); +} +``` + +Create a shared group route so both tabs can push common screens: + +```tsx +// app/(index,search)/_layout.tsx +import { Stack } from "expo-router/stack"; +import { PlatformColor } from "react-native"; + +export default function Layout({ segment }) { + const screen = segment.match(/\((.*)\)/)?.[1]!; + const titles: Record = { index: "Items", search: "Search" }; + + return ( + + + + + ); +} +``` diff --git a/examples/workbench/building-native-ui/suite.yml b/examples/workbench/building-native-ui/suite.yml new file mode 100644 index 0000000..df48bc6 --- /dev/null +++ b/examples/workbench/building-native-ui/suite.yml @@ -0,0 +1,44 @@ +name: building-native-ui-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4-6 + - openrouter/openai/gpt-5-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + Write all task outputs to the top level of /work unless told otherwise. + When reviewing code for violations, output one finding per line to findings.txt. +cases: + - name: review-media-player + task: | + You are reviewing a React Native / Expo screen component for guideline violations. + The Expo UI guidelines are in `references/building-native-ui/SKILL.md`. + + Review the file `MediaPlayerScreen.tsx` against every section of the guidelines: + Library Preferences, Responsiveness, Styling (including Shadows), Behavior, and Navigation. + + For each violation you find, write one line to `findings.txt` in this format: + MediaPlayerScreen.tsx:: + + Be thorough — check every import, every component, every style property. + graders: + - name: media-player-findings + command: node $CASE/checks/grade-media-player-findings.mjs + + - name: review-settings-screen + task: | + You are reviewing a React Native / Expo screen component for guideline violations. + The Expo UI guidelines are in `references/building-native-ui/SKILL.md`. + + Review the file `SettingsScreen.tsx` against every section of the guidelines: + Library Preferences, Responsiveness, Styling, Behavior, and Navigation. + + For each violation you find, write one line to `findings.txt` in this format: + SettingsScreen.tsx:: + + Be thorough — check every import, every component, every prop. + graders: + - name: settings-screen-findings + command: node $CASE/checks/grade-settings-screen-findings.mjs diff --git a/examples/workbench/building-native-ui/workspace/MediaPlayerScreen.tsx b/examples/workbench/building-native-ui/workspace/MediaPlayerScreen.tsx new file mode 100644 index 0000000..0bd02cf --- /dev/null +++ b/examples/workbench/building-native-ui/workspace/MediaPlayerScreen.tsx @@ -0,0 +1,39 @@ +import React from 'react'; +import { View, Text, Dimensions, Platform, SafeAreaView, StyleSheet } from 'react-native'; +import { Video } from 'expo-av'; + +const { width: screenWidth } = Dimensions.get('window'); + +export default function MediaPlayerScreen() { + const isIOS = Platform.OS === 'ios'; + + return ( + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: '#fff', + }, + card: { + padding: 16, + shadowColor: '#000', + shadowOffset: { width: 0, height: 1 }, + shadowOpacity: 0.1, + shadowRadius: 2, + elevation: 3, + borderRadius: 8, + }, +}); diff --git a/examples/workbench/building-native-ui/workspace/SettingsScreen.tsx b/examples/workbench/building-native-ui/workspace/SettingsScreen.tsx new file mode 100644 index 0000000..845bf23 --- /dev/null +++ b/examples/workbench/building-native-ui/workspace/SettingsScreen.tsx @@ -0,0 +1,24 @@ +import React, { useContext } from 'react'; +import { View, ScrollView, Text, Picker } from 'react-native'; +import Permissions from 'expo-permissions'; + +const ThemeContext = React.createContext({ dark: false }); + +export default function SettingsScreen() { + const theme = useContext(ThemeContext); + + return ( + + + Settings + + {}} + > + + + + + ); +} diff --git a/examples/workbench/firebase-auth-basics/README.md b/examples/workbench/firebase-auth-basics/README.md new file mode 100644 index 0000000..31f43fc --- /dev/null +++ b/examples/workbench/firebase-auth-basics/README.md @@ -0,0 +1,50 @@ +# firebase-auth-basics eval + +Eval suite for +[`firebase/agent-skills/firebase-auth-basics`](https://github.com/firebase/agent-skills) — +guide for setting up and using Firebase Authentication in web, Flutter, and Android apps. + +## Cases + +### `review-auth-js` — Web SDK auth patterns + +Sample: `workspace/src/auth.js` + +| Line | Violation | Rule | +|---|---|---| +| 4 | Missing `connectAuthEmulator` block for localhost | Connect to emulator when `location.hostname === "localhost"` | +| 11 | `auth.currentUser` used directly (synchronous, unreliable) | Use `onAuthStateChanged` to observe auth state | +| 17 | `createUserWithEmailAndPassword` missing `.catch` error handler | Auth calls must handle errors (errorCode, errorMessage) | +| 25 | `signInWithPopup` missing `try/catch` error handler | Auth calls must handle errors | + +### `review-firestore-rules` — Security rules + +Sample: `workspace/firestore.rules` + +| Line | Violation | Rule | +|---|---|---| +| 7 | Missing `request.auth != null` guard before `.uid` comparison (null-dereference) | Always check `request.auth != null` before accessing `.uid` | +| 11 | `allow read, write: if true` — no authentication required | Use `request.auth` checks to restrict access | + +## Vendored snapshot + +The skill normally references local markdown docs (`references/client_sdk_web.md`, +`references/security_rules.md`, etc.). These are already local-path references in +the upstream SKILL.md — no URL tweak is needed. We vendor the full reference set at +`references/firebase-auth-basics/references/` for eval determinism. Diff vs upstream +is zero lines (SKILL.md copied verbatim). + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-6` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/firebase-auth-basics/analysis.md b/examples/workbench/firebase-auth-basics/analysis.md new file mode 100644 index 0000000..7a390c1 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/analysis.md @@ -0,0 +1,18 @@ +--- +skill: firebase/agent-skills/firebase-auth-basics +status: success +classification: code-reviewer +baseline_rule_coverage: 1.00 +final_rule_coverage: 1.00 +modifications_tried: 0 +total_cost_usd: 0.60 +--- + +# Auto-pilot run for `firebase/agent-skills/firebase-auth-basics` + +- **Classification: code-reviewer** — skill prescribes Firebase Auth best practices (onAuthStateChanged, error handling, emulator connection, security rules); well-suited to seed-and-grade eval pattern. +- **Seeded 6 violations** across 2 files: `src/auth.js` (4 violations — missing emulator connection, direct `auth.currentUser` access, missing `.catch` on email sign-up, missing `try/catch` on Google sign-in) and `firestore.rules` (2 violations — missing `request.auth != null` null guard, unauthenticated write rule). +- **Baseline rule coverage: 1.00** on 16 completed trials (3 models × 2 cases × 3 trials = 18 total; 2 Gemini trials failed with transient SSE stream infrastructure errors unrelated to skill quality). +- **No modifications needed**: all models (Claude, GPT-5-mini, Gemini) reliably identified all seeded violations on first attempt; the skill's guidance is clear and effective. +- **No proposed upstream changes**: per lessons.md §"Don't manufacture problems", baseline ≥ 0.95 → exit clean. +- Total cost: ~$0.60 across 16 successful trials. diff --git a/examples/workbench/firebase-auth-basics/checks/_grader-utils.mjs b/examples/workbench/firebase-auth-basics/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/firebase-auth-basics/checks/grade-auth-findings.mjs b/examples/workbench/firebase-auth-basics/checks/grade-auth-findings.mjs new file mode 100644 index 0000000..82421a5 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/checks/grade-auth-findings.mjs @@ -0,0 +1,61 @@ +// Grader for firebase-auth-basics eval: checks findings.txt for violations +// in src/auth.js (4 seeded violations). +// +// Violations seeded in workspace/src/auth.js: +// V1 (~line 4): Missing connectAuthEmulator block for localhost +// V2 (~line 11): auth.currentUser used directly (should use onAuthStateChanged) +// V3 (~line 17): createUserWithEmailAndPassword without error handling (.catch) +// V4 (~line 25): signInWithPopup without try/catch error handling + +import { join } from 'node:path'; +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK ?? '/work'; +const findingsPath = join(WORK, 'findings.txt'); + +const expected = [ + { + id: 'V1-missing-emulator', + // connectAuthEmulator is missing near the getAuth call at line 4 + lines: looseRange(4, 10), + keywords: [ + fuzzyKeyword('connect auth emulator'), + tolerantKeyword('emulator'), + fuzzyKeyword('connectAuthEmulator'), + ], + }, + { + id: 'V2-currentUser-direct', + // auth.currentUser used directly at line 11 — should use onAuthStateChanged + lines: looseRange(11, 8), + keywords: [ + fuzzyKeyword('currentUser'), + fuzzyKeyword('onAuthStateChanged'), + fuzzyKeyword('auth state'), + ], + }, + { + id: 'V3-email-no-catch', + // createUserWithEmailAndPassword at line 17 missing error handling + lines: looseRange(17, 8), + keywords: [ + tolerantKeyword('error'), + tolerantKeyword('catch'), + fuzzyKeyword('createUserWithEmailAndPassword'), + fuzzyKeyword('error handling'), + ], + }, + { + id: 'V4-google-no-catch', + // signInWithPopup at line 25 missing try/catch + lines: looseRange(25, 8), + keywords: [ + tolerantKeyword('error'), + tolerantKeyword('catch'), + fuzzyKeyword('signInWithPopup'), + fuzzyKeyword('error handling'), + ], + }, +]; + +gradeFindings({ findingsPath, file: 'auth.js', expected }); diff --git a/examples/workbench/firebase-auth-basics/checks/grade-rules-findings.mjs b/examples/workbench/firebase-auth-basics/checks/grade-rules-findings.mjs new file mode 100644 index 0000000..cb0cb2f --- /dev/null +++ b/examples/workbench/firebase-auth-basics/checks/grade-rules-findings.mjs @@ -0,0 +1,44 @@ +// Grader for firebase-auth-basics eval: checks findings.txt for violations +// in firestore.rules (2 seeded violations). +// +// Violations seeded in workspace/firestore.rules: +// V5 (~line 7): Missing "request.auth != null" before .uid comparison +// V6 (~line 11): allow read, write: if true — no authentication required + +import { join } from 'node:path'; +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK ?? '/work'; +const findingsPath = join(WORK, 'findings.txt'); + +const expected = [ + { + id: 'V5-null-check-missing', + // request.auth.uid used without null guard at line 7 + lines: looseRange(7, 8), + keywords: [ + fuzzyKeyword('request.auth != null'), + fuzzyKeyword('null check'), + fuzzyKeyword('null guard'), + /request\.auth\s*!=\s*null/i, + tolerantKeyword('null'), + fuzzyKeyword('unauthenticated'), + fuzzyKeyword('NullPointer'), + ], + }, + { + id: 'V6-public-write', + // allow read, write: if true at line 11 — no auth + lines: looseRange(11, 8), + keywords: [ + fuzzyKeyword('if true'), + fuzzyKeyword('public'), + fuzzyKeyword('unauthenticated'), + fuzzyKeyword('no auth'), + tolerantKeyword('unrestricted'), + fuzzyKeyword('allow read, write'), + ], + }, +]; + +gradeFindings({ findingsPath, file: 'firestore.rules', expected }); diff --git a/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/SKILL.md b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/SKILL.md new file mode 100644 index 0000000..a1ed14d --- /dev/null +++ b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/SKILL.md @@ -0,0 +1,96 @@ +--- +name: firebase-auth-basics +description: Guide for setting up and using Firebase Authentication. Use this skill when the user's app requires user sign-in, user management, or secure data access using auth rules. +compatibility: This skill is best used with the Firebase CLI, but does not require it. Firebase CLI can be accessed through `npx -y firebase-tools@latest`. +--- + +## Prerequisites + +- **Firebase Project**: Created via `npx -y firebase-tools@latest projects:create` (see `firebase-basics`). +- **Firebase CLI**: Installed and logged in (see `firebase-basics`). + +## Core Concepts + +Firebase Authentication provides backend services, easy-to-use SDKs, and ready-made UI libraries to authenticate users to your app. + +### Users + +A user is an entity that can sign in to your app. Each user is identified by a unique ID (`uid`) which is guaranteed to be unique across all providers. +User properties include: +- `uid`: Unique identifier. +- `email`: User's email address (if available). +- `displayName`: User's display name (if available). +- `photoURL`: URL to user's photo (if available). +- `emailVerified`: Boolean indicating if the email is verified. + +### Identity Providers + +Firebase Auth supports multiple ways to sign in: +- **Email/Password**: Basic email and password authentication. +- **Federated Identity Providers**: Google, Facebook, Twitter, GitHub, Microsoft, Apple, etc. +- **Phone Number**: SMS-based authentication. +- **Anonymous**: Temporary guest accounts that can be linked to permanent accounts later. +- **Custom Auth**: Integrate with your existing auth system. + +Google Sign In is recommended as a good and secure default provider. + +### Tokens + +When a user signs in, they receive an ID Token (JWT). This token is used to identify the user when making requests to Firebase services (Realtime Database, Cloud Storage, Firestore) or your own backend. +- **ID Token**: Short-lived (1 hour), verifies identity. +- **Refresh Token**: Long-lived, used to get new ID tokens. + +## Workflow + +### 1. Provisioning + +#### Option 1. Enabling Authentication via CLI + +Only Google Sign In, anonymous auth, and email/password auth can be enabled via CLI. For other providers, use the Firebase Console. + +Configure Firebase Authentication in `firebase.json` by adding an 'auth' block: + +``` +{ + "auth": { + "providers": { + "anonymous": true, + "emailPassword": true, + "googleSignIn": { + "oAuthBrandDisplayName": "Your Brand Name", + "supportEmail": "support@example.com", + "authorizedRedirectUris": ["https://example.com"] + } + } + } +} +``` + +**CRITICAL**: After configuring `firebase.json`, you MUST deploy the auth configuration to the Firebase backend for the changes to take effect. This is essential for auth providers like Google Sign-In, email/password, etc. to auto-generate the necessary OAuth clients for your app platforms. Run: +```bash +npx -y firebase-tools@latest deploy --only auth +``` + +#### Option 2. Enabling Authentication in Console + +Enable other providers in the Firebase Console. + +1. Go to the https://console.firebase.google.com/project/_/authentication/providers +2. Select your project. +3. Enable the desired Sign-in providers (e.g., Email/Password, Google). + +### 2. Client Setup & Usage + +**Web** +See [references/client_sdk_web.md](references/client_sdk_web.md). + +**Flutter** +See [references/flutter_setup.md](references/flutter_setup.md). +**Android (Kotlin)** +See [references/client_sdk_android.md](references/client_sdk_android.md). + +### 3. Security Rules + +Secure your data using `request.auth` in Firestore/Storage rules. + +See [references/security_rules.md](references/security_rules.md). diff --git a/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_android.md b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_android.md new file mode 100644 index 0000000..a6dc882 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_android.md @@ -0,0 +1,157 @@ +# Firebase Authentication on Android (Kotlin) + +This guide walks you through using Firebase Authentication in your Android app using Kotlin DSL (`build.gradle.kts`) and Kotlin code. + +### 1, Enable Authentication via CLI + +Before adding dependencies in your app, make sure you enable the Auth service in your Firebase Project using the Firebase CLI: + +```bash +npx -y firebase-tools@latest init auth +``` + + --- + +### 2. Add Dependencies + +In your module-level `build.gradle.kts` (usually `app/build.gradle.kts`), add the dependency for Firebase Authentication: + +```kotlin +dependencies { + // [AGENT] Fetch the latest available BoM version from https://firebase.google.com/support/release-notes/android before adding this + implementation(platform("com.google.firebase:firebase-bom:")) + + // Add the dependency for the Firebase Authentication library + // When using the BoM, you don't specify versions in Firebase library dependencies + implementation("com.google.firebase:firebase-auth") +} +``` + +--- + +### 3. Initialize FirebaseAuth + +In your Activity or Fragment, initialize the `FirebaseAuth` instance: + +```kotlin +import com.google.firebase.auth.FirebaseAuth +import com.google.firebase.auth.ktx.auth +import com.google.firebase.ktx.Firebase + +class MainActivity : AppCompatActivity() { + + private lateinit var auth: FirebaseAuth + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val auth = Firebase.auth + + setContent { + MaterialTheme { + Text("Auth initialized!") + } + } + } +} +``` + +#### Jetpack Compose (Modern) + +Initialize inside a `ComponentActivity` using `setContent`: + +```kotlin +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import com.google.firebase.Firebase +import com.google.firebase.auth.auth + +class MainActivity : ComponentActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val auth = Firebase.auth + + setContent { + MaterialTheme { + Text("Auth initialized!") + } + } + } +} +``` + +--- + +### 4. Check Current Auth State + +You should check if a user is already signed in when your activity starts: + +```kotlin +public override fun onStart() { + super.onStart() + // Check if user is signed in (non-null) and update UI accordingly. + val currentUser = auth.currentUser + if (currentUser != null) { + // User is signed in, navigate to main screen or update UI + } else { + // No user is signed in, prompt for login + } +} +``` + +--- + +### 5. Sign Up New Users (Email/Password) + +Use `createUserWithEmailAndPassword` to register new users: + +```kotlin +fun signUpUser(email: String, password: String) { + auth.createUserWithEmailAndPassword(email, password) + .addOnCompleteListener(this) { task -> + if (task.isSuccessful) { + // Sign up success, update UI with the signed-in user's information + val user = auth.currentUser + // Navigate to main screen + } else { + // If sign up fails, display a message to the user. + Toast.makeText(baseContext, "Authentication failed.", Toast.LENGTH_SHORT).show() + } + } +} +``` + +--- + +### 6. Sign In Existing Users (Email/Password) + +Use `signInWithEmailAndPassword` to log in existing users: + +```kotlin +fun signInUser(email: String, password: String) { + auth.signInWithEmailAndPassword(email, password) + .addOnCompleteListener(this) { task -> + if (task.isSuccessful) { + // Sign in success, update UI with the signed-in user's information + val user = auth.currentUser + // Navigate to main screen + } else { + // If sign in fails, display a message to the user. + Toast.makeText(baseContext, "Authentication failed.", Toast.LENGTH_SHORT).show() + } + } +} +``` + +--- + +### 7. Sign Out + +To sign out a user, call `signOut()` on the `FirebaseAuth` instance: + +```kotlin +auth.signOut() +// Navigate to login screen +``` diff --git a/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_web.md b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_web.md new file mode 100644 index 0000000..493a66a --- /dev/null +++ b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/client_sdk_web.md @@ -0,0 +1,287 @@ +# Firebase Authentication Web SDK + +## Initialization + +First, ensure you have initialized the Firebase App (see `firebase-basics` skill). Then, initialize the Auth service: + +```javascript +import { getAuth } from "firebase/auth"; +import { app } from "./firebase"; // Your initialized Firebase App + +const auth = getAuth(app); +export { auth }; +``` + +## Connect to Emulator + +If you are running the Authentication emulator (usually on port 9099), connect to it immediately after initialization. + +```javascript +import { getAuth, connectAuthEmulator } from "firebase/auth"; + +const auth = getAuth(); +// Connect to emulator if running locally +if (location.hostname === "localhost") { + connectAuthEmulator(auth, "http://localhost:9099"); +} +``` + +## Sign Up with Email/Password + +```javascript +import { getAuth, createUserWithEmailAndPassword } from "firebase/auth"; + +const auth = getAuth(); +createUserWithEmailAndPassword(auth, email, password) + .then((userCredential) => { + const user = userCredential.user; + // ... + }) + .catch((error) => { + const errorCode = error.code; + const errorMessage = error.message; + // .. + }); +``` + +## Sign In with Google (Popup) + +```javascript +import { getAuth, signInWithPopup, GoogleAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new GoogleAuthProvider(); + +signInWithPopup(auth, provider) + .then((result) => { + // This gives you a Google Access Token. You can use it to access the Google API. + const credential = GoogleAuthProvider.credentialFromResult(result); + const token = credential.accessToken; + // The signed-in user info. + const user = result.user; + // ... + }) + .catch((error) => { + // Handle Errors here. + const errorCode = error.code; + const errorMessage = error.message; + // ... + }); +``` + +## Sign In with Facebook (Popup) + +```javascript +import { getAuth, signInWithPopup, FacebookAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new FacebookAuthProvider(); + +signInWithPopup(auth, provider) + .then((result) => { + // The signed-in user info. + const user = result.user; + // This gives you a Facebook Access Token. You can use it to access the Facebook API. + const credential = FacebookAuthProvider.credentialFromResult(result); + const accessToken = credential.accessToken; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In with Apple (Popup) + +```javascript +import { getAuth, signInWithPopup, OAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new OAuthProvider('apple.com'); + +signInWithPopup(auth, provider) + .then((result) => { + const user = result.user; + // Apple credential + const credential = OAuthProvider.credentialFromResult(result); + const accessToken = credential.accessToken; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In with Twitter (Popup) + +```javascript +import { getAuth, signInWithPopup, TwitterAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new TwitterAuthProvider(); + +signInWithPopup(auth, provider) + .then((result) => { + const user = result.user; + // Twitter credential + const credential = TwitterAuthProvider.credentialFromResult(result); + const token = credential.accessToken; + const secret = credential.secret; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In with GitHub (Popup) + +```javascript +import { getAuth, signInWithPopup, GithubAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new GithubAuthProvider(); + +signInWithPopup(auth, provider) + .then((result) => { + const user = result.user; + const credential = GithubAuthProvider.credentialFromResult(result); + const token = credential.accessToken; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In with Microsoft (Popup) + +```javascript +import { getAuth, signInWithPopup, OAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new OAuthProvider('microsoft.com'); + +signInWithPopup(auth, provider) + .then((result) => { + const user = result.user; + const credential = OAuthProvider.credentialFromResult(result); + const accessToken = credential.accessToken; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In with Yahoo (Popup) + +```javascript +import { getAuth, signInWithPopup, OAuthProvider } from "firebase/auth"; + +const auth = getAuth(); +const provider = new OAuthProvider('yahoo.com'); + +signInWithPopup(auth, provider) + .then((result) => { + const user = result.user; + const credential = OAuthProvider.credentialFromResult(result); + const accessToken = credential.accessToken; + }) + .catch((error) => { + // Handle Errors here. + }); +``` + +## Sign In Anonymously + +```javascript +import { getAuth, signInAnonymously } from "firebase/auth"; + +const auth = getAuth(); +signInAnonymously(auth) + .then(() => { + // Signed in.. + }) + .catch((error) => { + const errorCode = error.code; + const errorMessage = error.message; + }); +``` + +## Email Link Authentication + +**1. Send Auth Link** + +```javascript +import { getAuth, sendSignInLinkToEmail } from "firebase/auth"; + +const auth = getAuth(); +const actionCodeSettings = { + // URL you want to redirect back to. The domain must be in the authorized domains list in Firebase Console. + url: 'https://www.example.com/finishSignUp?cartId=1234', + handleCodeInApp: true, +}; + +sendSignInLinkToEmail(auth, email, actionCodeSettings) + .then(() => { + // Save the email locally so you don't need to ask the user for it again + window.localStorage.setItem('emailForSignIn', email); + }) + .catch((error) => { + // Error + }); +``` + +**2. Complete Sign In (on landing page)** + +```javascript +import { getAuth, isSignInWithEmailLink, signInWithEmailLink } from "firebase/auth"; + +const auth = getAuth(); + +if (isSignInWithEmailLink(auth, window.location.href)) { + let email = window.localStorage.getItem('emailForSignIn'); + if (!email) { + email = window.prompt('Please provide your email for confirmation'); + } + + signInWithEmailLink(auth, email, window.location.href) + .then((result) => { + window.localStorage.removeItem('emailForSignIn'); + // You can check result.user + }) + .catch((error) => { + // Error + }); +} +``` + +## Observe Auth State + +Recommended way to get the current user. This listener triggers whenever the user signs in or out. + +```javascript +import { getAuth, onAuthStateChanged } from "firebase/auth"; + +const auth = getAuth(); +onAuthStateChanged(auth, (user) => { + if (user) { + // User is signed in, see docs for a list of available properties + // https://firebase.google.com/docs/reference/js/firebase.User + const uid = user.uid; + // ... + } else { + // User is signed out + // ... + } +}); +``` + +## Sign Out + +```javascript +import { getAuth, signOut } from "firebase/auth"; + +const auth = getAuth(); +signOut(auth).then(() => { + // Sign-out successful. +}).catch((error) => { + // An error happened. +}); +``` diff --git a/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/flutter_setup.md b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/flutter_setup.md new file mode 100644 index 0000000..e1b7589 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/flutter_setup.md @@ -0,0 +1,98 @@ +# Firebase Auth & Google Sign-In for Flutter + +When integrating Firebase Authentication and Google Sign-In into Flutter apps targeting cross-platform environments (like Mobile + Web), you must navigate several breaking changes introduced in `google_sign_in` 7.x+ and some platform-specific quirks. + +## 1. `google_sign_in` 7.2.0 API Changes +- **Method Renamed**: The `signIn()` method is deprecated/removed and has been replaced with `authenticate()`. +- **Token Separation**: The `GoogleSignInAuthentication` object no longer packages both identity and authorization tokens together. Initial authentication now only provides the `idToken`. If an `accessToken` is required for Google APIs, you must explicitly request server authorization separately. + +## 2. Initialization & Web Hang/Crash Pitfalls +- **Initialization Requirement**: In 7.x, you must call `await GoogleSignIn.instance.initialize();` globally before using the plugin. +- **Web Client ID Constraint**: On Flutter Web, if you call `initialize()` without passing a `clientId` argument OR specifying the `` tag in `web/index.html`, the Dart Web Debug Service (DWDS) and the app will throw an assertion error and **hang infinitely**, resulting in a blank screen. +- **Common Workaround**: If you intend to use Firebase Auth's `signInWithPopup(GoogleAuthProvider())` for the web, you can conditionally skip the local `GoogleSignIn` package initialization entirely: + ```dart + import 'package:flutter/foundation.dart' show kIsWeb; + + if (!kIsWeb) { + await GoogleSignIn.instance.initialize(); + } + ``` + +## 3. Web Logout Crashes +- If you bypassed `GoogleSignIn` initialization on the web (as demonstrated above), you cannot call its `signOut()` method later. Attempting to execute `await GoogleSignIn.instance.signOut();` during the user's logout flow on the Web platform evaluates against an uninitialized context or unsupported environment, crashing the app. +- **Solution**: Conditionally separate the logout logic for Web to rely entirely on `FirebaseAuth`: + ```dart + if (!kIsWeb) { + await GoogleSignIn.instance.signOut(); + } + await FirebaseAuth.instance.signOut(); + ``` + +## 4. Prototyping Workaround: Bypassing Firestore Composite Indices +*Note: This is a Firestore consideration frequently encountered while fetching user-specific auth data.* + +When querying data via `FirebaseFirestore.instance`, using `.where('userId', isEqualTo: uid)` combined with a sort on a different field like `.orderBy('createdAt', descending: true)` mandates a custom composite index. +- **Quick Alternative**: During local development, you can avoid defining indexes by pulling the data using only `.where()` and applying the `.sort()` operation client-side on the resulting `List` in Dart. + +## 5. Robust `AuthService` Boilerplate +Here is a comprehensive `AuthService` implementation that properly handles the initialization and platform differences between Flutter Web and Mobile: + +```dart +import 'package:firebase_auth/firebase_auth.dart'; +import 'package:flutter/foundation.dart'; +import 'package:google_sign_in/google_sign_in.dart'; + +class AuthService { + final FirebaseAuth _auth = FirebaseAuth.instance; + + AuthService() { + if (!kIsWeb) { + GoogleSignIn.instance.initialize(); + } + } + + // Stream to listen to auth state changes + Stream get authStateChanges => _auth.authStateChanges(); + + // Get current user + User? get currentUser => _auth.currentUser; + + // Google Sign-In + Future signInWithGoogle() async { + try { + if (kIsWeb) { + // Web uses popup to avoid DWDS hangs and manual client ID config + GoogleAuthProvider authProvider = GoogleAuthProvider(); + return await _auth.signInWithPopup(authProvider); + } else { + // Mobile uses standard flow + final GoogleSignInAccount? googleUser = await GoogleSignIn.instance.authenticate(); + if (googleUser == null) return null; // Cancelled + + final GoogleSignInAuthentication googleAuth = await googleUser.authentication; + + final AuthCredential credential = GoogleAuthProvider.credential( + idToken: googleAuth.idToken, + ); + + return await _auth.signInWithCredential(credential); + } + } catch (e) { + print("Error during Google Sign-In: \$e"); + return null; + } + } + + // Sign out + Future signOut() async { + try { + if (!kIsWeb) { + await GoogleSignIn.instance.signOut(); + } + await _auth.signOut(); + } catch (e) { + print("Error signing out: \$e"); + } + } +} +``` diff --git a/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/security_rules.md b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/security_rules.md new file mode 100644 index 0000000..5de862a --- /dev/null +++ b/examples/workbench/firebase-auth-basics/references/firebase-auth-basics/references/security_rules.md @@ -0,0 +1,38 @@ +# Authentication in Security Rules + +Firebase Security Rules work with Firebase Authentication to provide rule-based access control. For better advice on writing safe security rules, +enable the `firebase-firestore-basics` or `firebase-storage-basics` skills. + +The `request.auth` variable contains authentication information for the user requesting data. + +## Basic Checks + +### Check if user is signed in +``` +allow read, write: if request.auth != null; +``` + +### Check if user owns the data +Access data only if the document ID matches the user's UID. +``` +allow read, write: if request.auth != null && request.auth.uid == userId; +``` +(Where `userId` is a path variable, e.g., `match /users/{userId}`) + +### Check if user owns the document (field-based) +Access data only if the document has a `owner_uid` field matching the user's UID. +``` +allow read, write: if request.auth != null && request.auth.uid == resource.data.owner_uid; +``` + +## Token Properties +`request.auth.token` contains standard JWT claims and custom claims. + +- `request.auth.token.email`: The user's email address. +- `request.auth.token.email_verified`: If the email is verified. +- `request.auth.token.name`: The user's display name. + +### Example: Email Verification Check +``` +allow create: if request.auth.token.email_verified == true; +``` diff --git a/examples/workbench/firebase-auth-basics/suite.yml b/examples/workbench/firebase-auth-basics/suite.yml new file mode 100644 index 0000000..d2640d4 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/suite.yml @@ -0,0 +1,46 @@ +name: firebase-auth-basics-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4.6 + - openrouter/openai/gpt-5-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 + +cases: + - name: review-auth-js + task: | + You have a Firebase Authentication implementation to review. The firebase-auth-basics + skill is available at firebase-auth-basics/SKILL.md (with supporting references + in firebase-auth-basics/references/). + + Review the file src/auth.js against the firebase-auth-basics guidelines and best + practices. Look for: + - Missing emulator connection for local development + - Incorrect auth state access patterns + - Missing error handling on auth calls + + Write every violation to findings.txt (create it at the top level of /work), one + per line. Each line must reference the filename and approximate line number, for example: + auth.js:11 - + graders: + - name: auth-findings + command: node $CASE/checks/grade-auth-findings.mjs + + - name: review-firestore-rules + task: | + You have a Firebase Authentication implementation to review. The firebase-auth-basics + skill is available at firebase-auth-basics/SKILL.md (with supporting references + in firebase-auth-basics/references/). + + Review the file firestore.rules against the firebase-auth-basics security rules + guidelines. Look for rules that are missing authentication checks or are + overly permissive. + + Write every violation to findings.txt (create it at the top level of /work), one + per line. Each line must reference the filename and approximate line number, for example: + firestore.rules:7 - + graders: + - name: rules-findings + command: node $CASE/checks/grade-rules-findings.mjs diff --git a/examples/workbench/firebase-auth-basics/workspace/firestore.rules b/examples/workbench/firebase-auth-basics/workspace/firestore.rules new file mode 100644 index 0000000..0b6bd9d --- /dev/null +++ b/examples/workbench/firebase-auth-basics/workspace/firestore.rules @@ -0,0 +1,16 @@ +rules_version = '2'; +service cloud.firestore { + match /databases/{database}/documents { + + // Users can access only their own data + match /users/{userId} { + allow read, write: if request.auth.uid == userId; + } + + // Posts are publicly readable and writable + match /posts/{postId} { + allow read, write: if true; + } + + } +} diff --git a/examples/workbench/firebase-auth-basics/workspace/src/auth.js b/examples/workbench/firebase-auth-basics/workspace/src/auth.js new file mode 100644 index 0000000..4dcd071 --- /dev/null +++ b/examples/workbench/firebase-auth-basics/workspace/src/auth.js @@ -0,0 +1,33 @@ +import { getAuth, createUserWithEmailAndPassword, signInWithPopup, GoogleAuthProvider } from "firebase/auth"; +import { app } from "./firebase"; + +const auth = getAuth(app); +// Emulator setup would go here if needed + +export { auth }; + +// Returns whether a user is currently logged in +export function isUserSignedIn() { + const user = auth.currentUser; + return user !== null; +} + +// Creates a new user account with email and password +export function signUpWithEmail(email, password) { + return createUserWithEmailAndPassword(auth, email, password) + .then((userCredential) => { + return userCredential.user; + }); +} + +// Signs in the user with their Google account +export async function signInWithGoogle() { + const provider = new GoogleAuthProvider(); + const result = await signInWithPopup(auth, provider); + return result.user; +} + +// Notifies the callback with the current user +export function observeAuthState(callback) { + callback(auth.currentUser); +} diff --git a/examples/workbench/firebase-hosting-basics/README.md b/examples/workbench/firebase-hosting-basics/README.md new file mode 100644 index 0000000..017c557 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/README.md @@ -0,0 +1,41 @@ +# firebase-hosting-basics eval + +Eval suite for +[`firebase/agent-skills/firebase-hosting-basics`](https://github.com/firebase/agent-skills) — +skill for working with Firebase Hosting (Classic): deploying static web apps and SPAs. + +## Cases + +### `review-firebase-config` — firebase.json best practices + +Sample: `workspace/firebase-app/firebase.json` + +| Line | Violation | Rule | +|---|---|---| +| 3 | `"public": "src"` — wrong directory; SPA builds output to `dist` or `build`, not `src` | `public` should point to build output dir | +| 5 | `ignore` list missing `**/.*` and `**/node_modules/**` patterns | Default ignores: `firebase.json`, `**/.*`, `**/node_modules/**` | +| 7 | `"cleanUrls": false` — should be `true` for clean URL paths | `cleanUrls` best practice is `true` | +| 12 | `"type": 200` — invalid redirect type; must be `301` or `302` | Redirects accept only `301` (permanent) or `302` (temporary) | +| 15–20 | No SPA catch-all rewrite `**` → `/index.html`; direct deep-links will return 404 | SPAs need `{ "source": "**", "destination": "/index.html" }` | + +## Vendored snapshot + +The skill normally reads `references/configuration.md` and `references/deploying.md` from the +same directory as `SKILL.md`. For deterministic eval we vendor a snapshot at +`references/firebase-hosting-basics/` and tweak `SKILL.md` to use local relative links +(removing the `references/` path prefix). Diff vs upstream is two lines. + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-6` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/firebase-hosting-basics/analysis.md b/examples/workbench/firebase-hosting-basics/analysis.md new file mode 100644 index 0000000..0589749 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/analysis.md @@ -0,0 +1,18 @@ +--- +skill: firebase/agent-skills/firebase-hosting-basics +status: success +classification: code-patterns +baseline_rule_coverage: 0.89 +final_rule_coverage: 1.00 +modifications_tried: 1 +total_cost_usd: 0.33 +--- + +# Auto-pilot run for `firebase/agent-skills/firebase-hosting-basics` + +- Classified as **code-patterns**: prescribes `firebase.json` configuration conventions (public dir, ignore, rewrites, cleanUrls, redirect types). Eval shaped as code-reviewer task: seed a misconfigured `firebase.json` with 5 known violations, ask agent to write findings to `findings.txt`, grade findings. +- Seeded 5 violations: wrong public dir (`src` not `dist`), incomplete ignore list (missing `**/.*` and `**/node_modules/**`), `cleanUrls: false`, invalid redirect type (`200` not `301/302`), missing SPA catch-all rewrite. +- **Grader calibration (iteration 0, not budgeted):** initial grader had too-tight line ranges for absence/redirect violations — Gemini reported violations at lines 6–8 while the actual lines were 12–20. Widened to `range(1, 22)` for those two; removed the generic `tolerantKeyword('missing')` keyword that caused a false positive on `incomplete-ignore`. Also tightened the task instructions to require `findings.txt` creation. +- Grader-calibrated baseline: 0.89 (Claude 3/3, Gemini 3/3, GPT-4o-mini 1/3). +- **Iteration 1:** GPT-4o-mini missed the SPA catch-all rewrite check (absence violation) and sometimes wrote an incomplete review (only 1 finding). Applied Recipe A (two-pass workflow) + Recipe E (rationale + consequence story) by adding `## Configuration Review` section to SKILL.md. GPT-4o-mini improved to 3/3; all valid trials passed. +- Final coverage: 1.00 (8/8 valid trials). One Gemini trial had a transient API error ("JSON error injected into SSE stream"), not a model behavior issue. diff --git a/examples/workbench/firebase-hosting-basics/checks/_grader-utils.mjs b/examples/workbench/firebase-hosting-basics/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/firebase-hosting-basics/checks/grade-firebase-config-findings.mjs b/examples/workbench/firebase-hosting-basics/checks/grade-firebase-config-findings.mjs new file mode 100644 index 0000000..b0a7f21 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/checks/grade-firebase-config-findings.mjs @@ -0,0 +1,59 @@ +// Grader for review-firebase-config case. +// Checks findings.txt for 5 known violations in firebase-app/firebase.json. +// +// Violation line map (1-indexed): +// Line 3: "public": "src" — wrong public dir for SPA (should be dist/build) +// Line 5: "firebase.json" — ignore list missing **/.* and **/node_modules/** +// Line 7: "cleanUrls": false — should be true +// Line 12: "type": 200 — invalid redirect type (must be 301 or 302) +// Lines 15-20: rewrites block — no SPA catch-all rewrite (** -> /index.html) + +import { gradeFindings, range, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const FINDINGS_PATH = `${process.env.WORK}/findings.txt`; + +// LLM line-counting notes for this file (22 lines): +// Presence violations (line clearly in file): use looseRange(N) default ±8. +// Absence/redirect violations: models report them at ~line 6-8 (the redirects/rewrites +// block header) rather than the actual type:200 line (12) or rewrites lines (15-20). +// Use range(1, 22) for those so any line in the file qualifies; rely on specific keywords. + +gradeFindings({ + findingsPath: FINDINGS_PATH, + file: 'firebase.json', + expected: [ + { + id: 'wrong-public-dir', + // Line 3: "public": "src" — models typically report lines 3-5 + lines: looseRange(3), + keywords: [/\bsrc\b/i, tolerantKeyword('public'), tolerantKeyword('dist'), tolerantKeyword('build'), tolerantKeyword('directory'), tolerantKeyword('output')], + }, + { + id: 'incomplete-ignore', + // Line 5: "firebase.json" only — models report lines 4-6; avoid 'missing' keyword + // to prevent false match with the SPA-rewrite finding ("Missing catch-all..."). + lines: looseRange(5), + keywords: [tolerantKeyword('ignore'), /node_modules/i, /\*\*\/\.\*/i, /pattern/i, /\.\*/, /dotfile/i], + }, + { + id: 'clean-urls-false', + // Line 7: "cleanUrls": false — models may report lines 2-8 due to drift + lines: looseRange(7), + keywords: [fuzzyKeyword('cleanUrls'), fuzzyKeyword('clean url'), tolerantKeyword('clean')], + }, + { + id: 'invalid-redirect-type', + // Line 12: "type": 200 — models often report at lines 6-8 (redirects block header); + // use whole-file range + highly specific keyword (\b200\b) to avoid cross-matches. + lines: range(1, 22), + keywords: [/\b200\b/, /must be 301/i, /must be 302/i, /invalid.*type/i, /type.*invalid/i], + }, + { + id: 'missing-spa-rewrite', + // Absence violation (no line to point at) — models report anywhere in lines 7-20; + // use whole-file range + keywords unique to this violation. + lines: range(1, 22), + keywords: [fuzzyKeyword('index.html'), /\bSPA\b/i, fuzzyKeyword('catch all'), fuzzyKeyword('single page'), /client.?side.?routing/i], + }, + ], +}); diff --git a/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/README.md b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/README.md new file mode 100644 index 0000000..ca4eca6 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/README.md @@ -0,0 +1,34 @@ +# Proposed upstream changes — firebase-hosting-basics + +## What changed + +Added a `## Configuration Review` section to `SKILL.md` implementing a **two-pass review workflow**. + +**Diff:** `firebase-agent-skills/before-SKILL.md` → `firebase-agent-skills/after-SKILL.md` +One new section appended at the end; zero changes to existing content. + +## Why (evidence from eval) + +Eval suite: `examples/workbench/firebase-hosting-basics/` — 1 case, 5 violations, 3 models × 3 trials. + +| Metric | Before | After | +|---|---|---| +| Rule-coverage (grader-calibrated baseline) | 0.89 | 1.00 | +| GPT-4o-mini pass rate | 1/3 | 3/3 | +| Gemini pass rate | 2/3* | 2/3* | +| Claude pass rate | 3/3 | 3/3 | + +*One Gemini trial failed with a transient API error ("JSON error injected into SSE stream"), not a model behavior issue. + +**Key failure pattern before the change:** +- GPT-4o-mini sometimes produced an incomplete review (only 1 finding) or missed the SPA catch-all rewrite check (absence violation). +- Absence violations are 5–10× harder for models than presence violations. The original SKILL.md had no guidance on what to look for when auditing an existing config. + +**What the new section adds:** +- Pass 1 explicitly lists incorrect literal values to scan for (presence violations). +- Pass 2 lists required-but-possibly-absent settings with concrete BAD consequences (missing `**/.*` exposes `.env`; missing catch-all rewrite causes 404s on deep links). + +## How to apply upstream + +Apply the diff to `skills/firebase-hosting-basics/SKILL.md` in the `firebase/agent-skills` repo. +The change is purely additive — no existing rules were modified or removed. diff --git a/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/after-SKILL.md b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/after-SKILL.md new file mode 100644 index 0000000..ead87fa --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/after-SKILL.md @@ -0,0 +1,78 @@ +--- +name: firebase-hosting-basics +description: Skill for working with Firebase Hosting (Classic). Use this when you want to deploy static web apps, Single Page Apps (SPAs), or simple microservices. Do NOT use for Firebase App Hosting. +--- + +# hosting-basics + +This skill provides instructions and references for working with Firebase Hosting, a fast and secure hosting service for your web app, static and dynamic content, and microservices. + +## Overview + +Firebase Hosting provides production-grade web content hosting for developers. With a single command, you can deploy web apps and serve both static and dynamic content to a global CDN (content delivery network). + +**Key Features:** +- **Fast Content Delivery:** Files are cached on SSDs at CDN edges around the world. +- **Secure by Default:** Zero-configuration SSL is built-in. +- **Preview Channels:** View and test changes on temporary preview URLs before deploying live. +- **GitHub Integration:** Automate previews and deploys with GitHub Actions. +- **Dynamic Content:** Serve dynamic content and microservices using Cloud Functions or Cloud Run. + +## Hosting vs App Hosting + +**Choose Firebase Hosting if:** +- You are deploying a static site (HTML/CSS/JS). +- You are deploying a simple SPA (React, Vue, etc. without SSR). +- You want full control over the build and deploy process via CLI. + +**Choose Firebase App Hosting if:** +- You are using a supported full-stack framework like Next.js or Angular. +- You need Server-Side Rendering (SSR) or ISR. +- You want an automated "git push to deploy" workflow with zero configuration. + +## Instructions + +### 1. Configuration (`firebase.json`) +For details on configuring Hosting behavior, including public directories, redirects, rewrites, and headers, see [configuration.md](references/configuration.md). + +### 2. Deploying +For instructions on deploying your site, using preview channels, and managing releases, see [deploying.md](references/deploying.md). + +### 3. Emulation +To test your app locally: +```bash +npx -y firebase-tools@latest emulators:start --only hosting +``` +This serves your app at `http://localhost:5000` by default. + +## Configuration Review + +When auditing a `firebase.json` for compliance and best practices, review in **two passes** — both are required. + +### Pass 1 — Visible bad values + +Scan each key for incorrect values: + +- `"public"`: must point to the **build output directory** (`dist` or `build`), NOT the source directory (`src`). Using `src` deploys unbuilt source files. +- `"cleanUrls"`: should be `true`. Setting it to `false` exposes `.html` extensions in all URLs. +- Redirect `"type"`: must be `301` (permanent) or `302` (temporary). The value `200` is **not a valid redirect type** and will cause errors. + +### Pass 2 — Required but absent settings + +The most-missed issues are about settings that should be present but are **missing entirely**. After Pass 1, check each section: + +**`ignore` array** — must include all three default patterns: + +```json +"ignore": ["firebase.json", "**/.*", "**/node_modules/**"] +``` + +Missing `**/.*` exposes hidden files (`.env`, `.htaccess`). Missing `**/node_modules/**` uploads tens of thousands of dependency files. + +**SPA catch-all rewrite** — if the project is a Single Page Application (React, Vue, Angular, etc.), the `rewrites` array MUST contain a catch-all rule: + +```json +{ "source": "**", "destination": "/index.html" } +``` + +Without this rule, direct navigation to any deep link (e.g., `/dashboard`, `/profile/42`) returns a `404 Not Found` error from the CDN because no matching file exists. Client-side routing only works when the app is served from `index.html`. diff --git a/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/before-SKILL.md b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/before-SKILL.md new file mode 100644 index 0000000..a83ac28 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/proposed-upstream-changes/firebase-agent-skills/before-SKILL.md @@ -0,0 +1,46 @@ +--- +name: firebase-hosting-basics +description: Skill for working with Firebase Hosting (Classic). Use this when you want to deploy static web apps, Single Page Apps (SPAs), or simple microservices. Do NOT use for Firebase App Hosting. +--- + +# hosting-basics + +This skill provides instructions and references for working with Firebase Hosting, a fast and secure hosting service for your web app, static and dynamic content, and microservices. + +## Overview + +Firebase Hosting provides production-grade web content hosting for developers. With a single command, you can deploy web apps and serve both static and dynamic content to a global CDN (content delivery network). + +**Key Features:** +- **Fast Content Delivery:** Files are cached on SSDs at CDN edges around the world. +- **Secure by Default:** Zero-configuration SSL is built-in. +- **Preview Channels:** View and test changes on temporary preview URLs before deploying live. +- **GitHub Integration:** Automate previews and deploys with GitHub Actions. +- **Dynamic Content:** Serve dynamic content and microservices using Cloud Functions or Cloud Run. + +## Hosting vs App Hosting + +**Choose Firebase Hosting if:** +- You are deploying a static site (HTML/CSS/JS). +- You are deploying a simple SPA (React, Vue, etc. without SSR). +- You want full control over the build and deploy process via CLI. + +**Choose Firebase App Hosting if:** +- You are using a supported full-stack framework like Next.js or Angular. +- You need Server-Side Rendering (SSR) or ISR. +- You want an automated "git push to deploy" workflow with zero configuration. + +## Instructions + +### 1. Configuration (`firebase.json`) +For details on configuring Hosting behavior, including public directories, redirects, rewrites, and headers, see [configuration.md](references/configuration.md). + +### 2. Deploying +For instructions on deploying your site, using preview channels, and managing releases, see [deploying.md](references/deploying.md). + +### 3. Emulation +To test your app locally: +```bash +npx -y firebase-tools@latest emulators:start --only hosting +``` +This serves your app at `http://localhost:5000` by default. diff --git a/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/SKILL.md b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/SKILL.md new file mode 100644 index 0000000..f392856 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/SKILL.md @@ -0,0 +1,78 @@ +--- +name: firebase-hosting-basics +description: Skill for working with Firebase Hosting (Classic). Use this when you want to deploy static web apps, Single Page Apps (SPAs), or simple microservices. Do NOT use for Firebase App Hosting. +--- + +# hosting-basics + +This skill provides instructions and references for working with Firebase Hosting, a fast and secure hosting service for your web app, static and dynamic content, and microservices. + +## Overview + +Firebase Hosting provides production-grade web content hosting for developers. With a single command, you can deploy web apps and serve both static and dynamic content to a global CDN (content delivery network). + +**Key Features:** +- **Fast Content Delivery:** Files are cached on SSDs at CDN edges around the world. +- **Secure by Default:** Zero-configuration SSL is built-in. +- **Preview Channels:** View and test changes on temporary preview URLs before deploying live. +- **GitHub Integration:** Automate previews and deploys with GitHub Actions. +- **Dynamic Content:** Serve dynamic content and microservices using Cloud Functions or Cloud Run. + +## Hosting vs App Hosting + +**Choose Firebase Hosting if:** +- You are deploying a static site (HTML/CSS/JS). +- You are deploying a simple SPA (React, Vue, etc. without SSR). +- You want full control over the build and deploy process via CLI. + +**Choose Firebase App Hosting if:** +- You are using a supported full-stack framework like Next.js or Angular. +- You need Server-Side Rendering (SSR) or ISR. +- You want an automated "git push to deploy" workflow with zero configuration. + +## Instructions + +### 1. Configuration (`firebase.json`) +For details on configuring Hosting behavior, including public directories, redirects, rewrites, and headers, see [configuration.md](configuration.md). + +### 2. Deploying +For instructions on deploying your site, using preview channels, and managing releases, see [deploying.md](deploying.md). + +### 3. Emulation +To test your app locally: +```bash +npx -y firebase-tools@latest emulators:start --only hosting +``` +This serves your app at `http://localhost:5000` by default. + +## Configuration Review + +When auditing a `firebase.json` for compliance and best practices, review in **two passes** — both are required. + +### Pass 1 — Visible bad values + +Scan each key for incorrect values: + +- `"public"`: must point to the **build output directory** (`dist` or `build`), NOT the source directory (`src`). Using `src` deploys unbuilt source files. +- `"cleanUrls"`: should be `true`. Setting it to `false` exposes `.html` extensions in all URLs. +- Redirect `"type"`: must be `301` (permanent) or `302` (temporary). The value `200` is **not a valid redirect type** and will cause errors. + +### Pass 2 — Required but absent settings + +The most-missed issues are about settings that should be present but are **missing entirely**. After Pass 1, check each section: + +**`ignore` array** — must include all three default patterns: + +```json +"ignore": ["firebase.json", "**/.*", "**/node_modules/**"] +``` + +Missing `**/.*` exposes hidden files (`.env`, `.htaccess`). Missing `**/node_modules/**` uploads tens of thousands of dependency files. + +**SPA catch-all rewrite** — if the project is a Single Page Application (React, Vue, Angular, etc.), the `rewrites` array MUST contain a catch-all rule: + +```json +{ "source": "**", "destination": "/index.html" } +``` + +Without this rule, direct navigation to any deep link (e.g., `/dashboard`, `/profile/42`) returns a `404 Not Found` error from the CDN because no matching file exists. Client-side routing only works when the app is served from `index.html`. diff --git a/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/configuration.md b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/configuration.md new file mode 100644 index 0000000..cdeddc9 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/configuration.md @@ -0,0 +1,115 @@ +# Hosting Configuration (`firebase.json`) + +The `hosting` section of `firebase.json` configures how your site is deployed and served. + +## Key Attributes + +### `public` (Required) +Specifies the directory to deploy to Firebase Hosting. For most SPA frameworks (React, Vue, Angular), this should point to the **build output directory** (`dist` or `build`), NOT the source directory (`src`). + +```json +"hosting": { + "public": "dist" +} +``` + +### `ignore` (Optional) +Files to ignore on deploy. Uses glob patterns (like `.gitignore`). +**Default ignores:** `firebase.json`, `**/.*`, `**/node_modules/**` + +You should always include at minimum: +```json +"ignore": [ + "firebase.json", + "**/.*", + "**/node_modules/**" +] +``` + +### `redirects` (Optional) +URL redirects to prevent broken links or shorten URLs. The `type` field must be `301` (permanent) or `302` (temporary). **`200` is NOT a valid redirect type.** + +```json +"redirects": [ + { + "source": "/foo", + "destination": "/bar", + "type": 301 + } +] +``` + +### `rewrites` (Optional) +Serve the same content for multiple URLs, useful for SPAs or Dynamic Content. + +**For Single Page Applications (SPAs):** You must include a catch-all rewrite that routes all unmatched paths to `/index.html`, otherwise direct navigation to deep links will return 404. + +```json +"rewrites": [ + { + "source": "**", + "destination": "/index.html" + }, + { + "source": "/api/**", + "function": "apiFunction" + }, + { + "source": "/container/**", + "run": { + "serviceId": "helloworld", + "region": "us-central1" + } + } +] +``` + +### `headers` (Optional) +Custom response headers. +```json +"headers": [ + { + "source": "**/*.@(eot|otf|ttf|ttc|woff|font.css)", + "headers": [ + { + "key": "Access-Control-Allow-Origin", + "value": "*" + } + ] + } +] +``` + +### `cleanUrls` (Optional) +If `true`, drops `.html` extension from URLs. **Best practice: set to `true`.** +```json +"cleanUrls": true +``` + +### `trailingSlash` (Optional) +Controls trailing slashes in static content URLs. +- `true`: Adds trailing slash. +- `false`: Removes trailing slash. + +## Full Example + +```json +{ + "hosting": { + "public": "dist", + "ignore": [ + "firebase.json", + "**/.*", + "**/node_modules/**" + ], + "rewrites": [ + { + "source": "**", + "destination": "/index.html" + } + ], + "cleanUrls": true, + "trailingSlash": false + } +} +``` diff --git a/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/deploying.md b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/deploying.md new file mode 100644 index 0000000..db57409 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/references/firebase-hosting-basics/deploying.md @@ -0,0 +1,30 @@ +# Firebase Hosting Deployment Guide + +## Standard Deployment + +Execute `npx -y firebase-tools@latest deploy --only hosting` to push content to your default sites at `PROJECT_ID.web.app` and `PROJECT_ID.firebaseapp.com`. + +## Preview Channels + +Preview channels allow you to test changes on a temporary URL before going live. + +Deploy using: +```bash +npx -y firebase-tools@latest hosting:channel:deploy CHANNEL_ID +``` + +This generates a preview URL like `PROJECT_ID--CHANNEL_ID-RANDOM_HASH.web.app`. + +By default, channels expire after 7 days unless you customize the timeframe using the `--expires` flag (e.g., `--expires 1d`). + +## Promoting to Live + +Use the cloning command to move a preview channel version to production without rebuilding: +```bash +npx -y firebase-tools@latest hosting:clone SOURCE_SITE_ID:SOURCE_CHANNEL_ID TARGET_SITE_ID:live +``` + +For example, to promote a `feature-beta` channel to your live site: +```bash +npx -y firebase-tools@latest hosting:clone my-app:feature-beta my-app:live +``` diff --git a/examples/workbench/firebase-hosting-basics/suite.yml b/examples/workbench/firebase-hosting-basics/suite.yml new file mode 100644 index 0000000..bbb4e0f --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/suite.yml @@ -0,0 +1,36 @@ +name: firebase-hosting-basics-eval +references: ./references +workspace: ./workspace +models: + - openrouter/anthropic/claude-sonnet-4-6 + - openrouter/openai/gpt-4o-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + Write all output files at the top level of /work unless the task specifies otherwise. +cases: + - name: review-firebase-config + task: | + You are a Firebase Hosting configuration expert. + + Review the file `firebase-app/firebase.json` for configuration problems and best-practice violations. + + Use the Firebase Hosting skill documentation in `references/firebase-hosting-basics/SKILL.md` to guide your review. + Read `references/firebase-hosting-basics/configuration.md` and `references/firebase-hosting-basics/deploying.md` for detailed reference. + + For each issue you find, append a line to `findings.txt` in the format: + firebase.json:: + + You MUST create `findings.txt` and write every issue to it before finishing. + + Check for ALL of the following: + 1. Correct public directory for SPA builds (should be dist or build, not src) + 2. Complete ignore patterns (should include firebase.json, **/.*, **/node_modules/**) + 3. Proper cleanUrls setting (should be true for clean URLs) + 4. Valid redirect types (must be 301 or 302 — the value 200 is NOT valid for redirects) + 5. SPA catch-all rewrite rule routing all paths to /index.html (required for client-side routing) + graders: + - name: firebase-config-findings + command: node $CASE/checks/grade-firebase-config-findings.mjs diff --git a/examples/workbench/firebase-hosting-basics/workspace/firebase-app/firebase.json b/examples/workbench/firebase-hosting-basics/workspace/firebase-app/firebase.json new file mode 100644 index 0000000..ab1b781 --- /dev/null +++ b/examples/workbench/firebase-hosting-basics/workspace/firebase-app/firebase.json @@ -0,0 +1,22 @@ +{ + "hosting": { + "public": "src", + "ignore": [ + "firebase.json" + ], + "cleanUrls": false, + "redirects": [ + { + "source": "/old-blog", + "destination": "/blog", + "type": 200 + } + ], + "rewrites": [ + { + "source": "/api/**", + "function": "apiFunction" + } + ] + } +} diff --git a/examples/workbench/firecrawl-build-scrape/README.md b/examples/workbench/firecrawl-build-scrape/README.md new file mode 100644 index 0000000..145a89a --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/README.md @@ -0,0 +1,41 @@ +# firecrawl-build-scrape eval + +Eval suite for +[`firecrawl/skills/firecrawl-build-scrape`](https://github.com/firecrawl/skills) — +integrate Firecrawl `/scrape` into product code for single-page extraction. + +## Cases + +### `review-scrape-integration` — Firecrawl scrape pattern violations + +Sample: `workspace/ScrapeService.ts` + +| Line | Violation | Rule | +|---|---|---| +| 10 | `scrapeArticle` missing `onlyMainContent: true` | "Use `onlyMainContent` for article-like pages where nav and chrome add noise." | +| 18 | `scrapeCompanyPage` uses `formats: ['html']` instead of markdown | "Return `markdown` unless the feature truly needs another format." | +| 27 | `scrapeNews` uses `waitFor: 5000` on a static news site | "Add waits or other rendering options only when the page needs them." | +| 34 | `findAndScrapeCompany` passes a search query string to `/scrape` instead of a URL | "If you do not have the URL yet, start with the search skill." | +| 43 | `scrapeDocPage` requests 4 formats (`markdown`, `html`, `links`, `screenshot`) | "Keep the integration narrow: one feature, one URL, one extraction contract." | + +## Vendored snapshot + +The skill normally fetches docs from `docs.firecrawl.dev/agent-source-of-truth/`. +For deterministic eval we vendor a Node.js snapshot at +`references/firecrawl-build-scrape/node-docs.md` and tweak `SKILL.md` to read it +locally. Diff vs upstream is one line (the Node/TypeScript docs URL). + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-5` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-flash` diff --git a/examples/workbench/firecrawl-build-scrape/analysis.md b/examples/workbench/firecrawl-build-scrape/analysis.md new file mode 100644 index 0000000..b6bc13e --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/analysis.md @@ -0,0 +1,18 @@ +--- +skill: firecrawl/skills/firecrawl-build-scrape +status: uplift-too-small +classification: code-patterns +baseline_rule_coverage: 0.84 +final_rule_coverage: 0.89 +modifications_tried: 2 +total_cost_usd: 0.17 +--- + +# Auto-pilot run for `firecrawl/skills/firecrawl-build-scrape` + +- Classified as **code-patterns**: the skill prescribes how to integrate Firecrawl `/scrape` — markdown default, `onlyMainContent` for articles, no unnecessary `waitFor`, escalate to search when URL is unknown, keep contracts narrow. +- Seeded `ScrapeService.ts` with 5 violations: missing `onlyMainContent` in article scraper (absence), wrong `html` format (presence), unnecessary `waitFor` (presence), query-not-URL escalation (escalation), too-many-formats (design). Each maps directly to an explicit skill rule. +- Baseline: 38/45 = 0.844 — V1 (missing onlyMainContent) was the highest-miss rule (44%), V4 (escalation) at 22%. +- Iteration 1: Added per-pattern integration checklist (Recipe C) and BAD/GOOD escalation example (Recipe E). Improved sonnet fully; gemini already perfect. gpt-4o-mini unchanged. 39/45 = 0.867 (+0.022). +- Iteration 2: Added BAD/GOOD code example for V1 (missing onlyMainContent) directly in the checklist. Marginal improvement for gpt-4o-mini. 40/45 = 0.889 (+0.044 from baseline — below +0.05 threshold). +- gpt-4o-mini consistently emits only 3–4 findings instead of 5, skipping scrapeArticle. Remaining gap is model-capability, not skill-wording — modifications are additive and benefit sonnet/gemini fully. diff --git a/examples/workbench/firecrawl-build-scrape/checks/_grader-utils.mjs b/examples/workbench/firecrawl-build-scrape/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/firecrawl-build-scrape/checks/grade-scrape-service-findings.mjs b/examples/workbench/firecrawl-build-scrape/checks/grade-scrape-service-findings.mjs new file mode 100644 index 0000000..129e34f --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/checks/grade-scrape-service-findings.mjs @@ -0,0 +1,86 @@ +// Grader for review-scrape-integration case. +// Checks that findings.txt identifies the 5 deliberate violations in +// workspace/ScrapeService.ts against the firecrawl-build-scrape skill rules. + +import { join } from 'node:path'; +import { + gradeFindings, + looseRange, + fuzzyKeyword, + tolerantKeyword, +} from './_grader-utils.mjs'; + +const findingsPath = join(process.env.WORK, 'findings.txt'); + +// ScrapeService.ts violation map (line numbers verified against workspace/ScrapeService.ts): +// +// V1 (line ~10): scrapeArticle — missing onlyMainContent for article-like page +// V2 (line ~18): scrapeCompanyPage — uses 'html' format instead of markdown default +// V3 (line ~27): scrapeNews — unnecessary waitFor: 5000 on a static news site +// V4 (line ~34): findAndScrapeCompany — passes query string instead of URL (should escalate to search) +// V5 (line ~43): scrapeDocPage — requests too many formats; violates narrow-contract rule + +const expected = [ + { + id: 'V1-missing-onlyMainContent', + // Absence violation: looseRange with wider tolerance so function-level references match + lines: looseRange(10, 10), + keywords: [ + tolerantKeyword('onlyMainContent'), + fuzzyKeyword('main content'), + tolerantKeyword('article'), + tolerantKeyword('nav'), + tolerantKeyword('noise'), + ], + }, + { + id: 'V2-html-format', + lines: looseRange(18, 8), + keywords: [ + tolerantKeyword('html'), + tolerantKeyword('format'), + tolerantKeyword('markdown'), + ], + }, + { + id: 'V3-unnecessary-waitFor', + lines: looseRange(27, 8), + keywords: [ + tolerantKeyword('waitFor'), + tolerantKeyword('wait'), + tolerantKeyword('render'), + tolerantKeyword('static'), + tolerantKeyword('unnecessar'), + ], + }, + { + id: 'V4-query-not-url', + // Escalation violation: model may reference function declaration or scrape call + lines: looseRange(34, 10), + keywords: [ + tolerantKeyword('search'), + tolerantKeyword('escalat'), + fuzzyKeyword('search skill'), + tolerantKeyword('URL'), + tolerantKeyword('query'), + ], + }, + { + id: 'V5-too-broad-contract', + lines: looseRange(43, 10), + keywords: [ + tolerantKeyword('narrow'), + tolerantKeyword('screenshot'), + fuzzyKeyword('multiple format'), + tolerantKeyword('contract'), + tolerantKeyword('broad'), + tolerantKeyword('format'), + ], + }, +]; + +gradeFindings({ + findingsPath, + file: 'ScrapeService.ts', + expected, +}); diff --git a/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/SKILL.md b/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/SKILL.md new file mode 100644 index 0000000..51f7844 --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/SKILL.md @@ -0,0 +1,117 @@ +--- +name: firecrawl-build-scrape +description: Integrate Firecrawl `/scrape` into product code for single-page extraction. Use when an app already has a URL and needs markdown, HTML, links, screenshots, metadata, or structured page output. Prefer this skill over broader crawl patterns when the feature is page-level. +license: ISC +metadata: + author: firecrawl + version: "0.1.0" + homepage: https://www.firecrawl.dev + source: https://github.com/firecrawl/skills +inputs: + - name: FIRECRAWL_API_KEY + description: Firecrawl API key for hosted Firecrawl requests. + required: true + - name: FIRECRAWL_API_URL + description: Optional base URL for self-hosted Firecrawl deployments. + required: false +--- + +# Firecrawl Build Scrape + +Use this when the application already has the URL and needs content from one page. + +## Use This When + +- the feature starts from a known URL +- you need page content for retrieval, summarization, enrichment, or monitoring +- you want the default extraction primitive before considering `/interact` + +## Default Recommendations + +- Return `markdown` unless the feature truly needs another format. +- Use `onlyMainContent` for article-like pages where nav and chrome add noise. +- Add waits or other rendering options only when the page needs them. + +## Common Product Patterns + +- knowledge ingestion from known URLs +- enrichment from a company, product, or docs page +- pricing, changelog, and documentation extraction +- page-level quality checks or monitoring + +## Escalation Rules + +- If you do not have the URL yet, start with the search skill (`firecrawl-build-search`). +- If content requires clicks, typing, or multi-step navigation, escalate to the interact skill (`firecrawl-build-interact`). + +## Implementation Notes + +- Keep the integration narrow: one feature, one URL, one extraction contract. +- Treat `/scrape` as the default primitive for downstream LLM or indexing pipelines. +- Request richer formats only when the consumer needs them, such as links, screenshots, or branding data. + +## Integration Checklist + +Run this checklist on every `/scrape` integration before finalizing. + +### Every article / blog / news function + +- [ ] `onlyMainContent: true` is set — this removes nav, sidebar, and footer noise from article-like pages. If it is absent, the consumer receives noisy HTML-derived markdown. +- [ ] `formats: ['markdown']` — default unless the consumer explicitly needs another format. +- [ ] No `waitFor` unless the page is confirmed to be a SPA or has lazy-loaded content. + +**BAD** — article scraper missing `onlyMainContent` (nav menus, sidebars, and footers contaminate the markdown): +```ts +export async function scrapeArticle(url: string) { + const doc = await client.scrape(url, { + formats: ['markdown'], + // BUG: nav + sidebar noise included in output + }); + return doc.markdown; +} +``` + +**GOOD** — article scraper with `onlyMainContent: true`: +```ts +export async function scrapeArticle(url: string) { + const doc = await client.scrape(url, { + formats: ['markdown'], + onlyMainContent: true, // strips nav, sidebar, footer + }); + return doc.markdown; +} + +### Every enrichment / company / docs function + +- [ ] `formats: ['markdown']` — HTML is only justified when a downstream parser requires raw HTML. +- [ ] `waitFor` is absent or explicitly justified in a comment. +- [ ] Each format in the `formats` array has a named consumer — remove formats nobody reads. + +### Escalation check (MUST run before every scrape call) + +> NEVER pass a search query string to `client.scrape()`. The `/scrape` endpoint requires a fully-formed URL. If the calling code receives a keyword, topic, or company name rather than a URL, it MUST first call `client.search()` (firecrawl-build-search skill) to resolve a URL, then pass that URL to `/scrape`. + +**BAD** — query string passed directly to `/scrape`: +```ts +// The caller has a company name, not a URL. +const doc = await client.scrape(companyName, { formats: ['markdown'] }); +``` + +**GOOD** — search first, then scrape: +```ts +// Resolve URL first with client.search(), then scrape. +const results = await client.search(companyName, { limit: 1 }); +const doc = await client.scrape(results[0].url, { formats: ['markdown'] }); +``` + +## Docs (Source of Truth) + +Read the source-of-truth reference for your project language before writing integration code: + +- **Node / TypeScript**: See `references/firecrawl-build-scrape/node-docs.md` + +## See Also + +- firecrawl-build +- firecrawl-build-search +- firecrawl-build-interact diff --git a/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/node-docs.md b/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/node-docs.md new file mode 100644 index 0000000..ac1ba03 --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/references/firecrawl-build-scrape/node-docs.md @@ -0,0 +1,81 @@ +# Firecrawl Node.js / TypeScript — Source of Truth + +## Installation + +```bash +npm install @mendable/firecrawl-js +``` + +## Authentication + +```ts +import FirecrawlApp from '@mendable/firecrawl-js'; + +const client = new FirecrawlApp({ + apiKey: process.env.FIRECRAWL_API_KEY, +}); +``` + +## Basic Scrape + +```ts +const doc = await client.scrape('https://docs.firecrawl.dev', { + formats: ['markdown'], +}); +console.log(doc.markdown); +``` + +## Method Signature + +`client.scrape(url: string, options?: ScrapeParams): Promise` + +## Key Options + +| Option | Type | Default | Notes | +|---|---|---|---| +| `formats` | `string[]` | `['markdown']` | Requested output formats. Default is markdown. | +| `onlyMainContent` | `boolean` | `false` | Strip nav, footer, sidebars. Use for articles and blog posts. | +| `waitFor` | `number` | `0` | Milliseconds to wait for JS rendering. Use **only** when the page requires it (SPA, lazy-loaded content). | +| `includeTags` | `string[]` | — | Include only these HTML tags in processing. | +| `excludeTags` | `string[]` | — | Strip these HTML tags before processing. | +| `timeout` | `number` | 30000 | Request timeout in ms. | +| `mobile` | `boolean` | `false` | Use mobile viewport. | +| `blockAds` | `boolean` | `false` | Block ads and popups. | +| `proxy` | `string` | — | `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or custom URL. | + +## Format Values + +Plain string formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"audio"`, `"branding"` + +Object formats (structured extraction): +```ts +{ type: 'json', prompt?: string, schema?: JSONSchema } +{ type: 'question', question: string } +{ type: 'highlights', query: string } +{ type: 'screenshot', fullPage?: boolean, quality?: number } +``` + +## Pattern: Article Extraction + +```ts +// Use onlyMainContent for article-like pages to avoid nav noise. +const doc = await client.scrape(url, { + formats: ['markdown'], + onlyMainContent: true, +}); +``` + +## Pattern: Structured Data Extraction + +```ts +// Request richer formats only when the consumer needs them. +const doc = await client.scrape(url, { + formats: ['markdown', { type: 'json', prompt: 'Extract plan names and prices.' }], +}); +``` + +## When NOT to Use This + +- You don't have the URL yet → use `client.search()` first +- Page requires user interactions (clicks, forms) → use `client.interact()` +- You need to crawl multiple pages → use `client.crawl()` diff --git a/examples/workbench/firecrawl-build-scrape/suite.yml b/examples/workbench/firecrawl-build-scrape/suite.yml new file mode 100644 index 0000000..34f16c6 --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/suite.yml @@ -0,0 +1,26 @@ +name: firecrawl-build-scrape-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4-5 + - openrouter/openai/gpt-4o-mini + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 + +cases: + - name: review-scrape-integration + task: | + You have the firecrawl-build-scrape skill at references/firecrawl-build-scrape/SKILL.md and the + Node.js API reference at references/firecrawl-build-scrape/node-docs.md. + + Review the file ScrapeService.ts against the patterns and rules described in the skill. + Find every violation of the skill's guidelines. + + Write all violations to findings.txt, one per line, in this exact format: + ScrapeService.ts: - + + Be specific about line numbers. Include every violation you find. + graders: + - name: scrape-service-findings + command: node $CASE/checks/grade-scrape-service-findings.mjs diff --git a/examples/workbench/firecrawl-build-scrape/workspace/ScrapeService.ts b/examples/workbench/firecrawl-build-scrape/workspace/ScrapeService.ts new file mode 100644 index 0000000..c6a0871 --- /dev/null +++ b/examples/workbench/firecrawl-build-scrape/workspace/ScrapeService.ts @@ -0,0 +1,52 @@ +import FirecrawlApp from '@mendable/firecrawl-js'; + +const client = new FirecrawlApp({ + apiKey: process.env.FIRECRAWL_API_KEY, +}); + +// Scrape a blog article and return its content. +export async function scrapeArticle(url: string): Promise { + const doc = await client.scrape(url, { + formats: ['markdown'], + }); + return doc.markdown ?? ''; +} + +// Scrape a company's homepage for CRM enrichment data. +export async function scrapeCompanyPage(url: string): Promise { + const doc = await client.scrape(url, { + formats: ['html'], + }); + return doc.html ?? ''; +} + +// Scrape a news article for content monitoring. +export async function scrapeNews(url: string): Promise { + const doc = await client.scrape(url, { + formats: ['markdown'], + waitFor: 5000, + }); + return doc.markdown ?? ''; +} + +// Find and scrape a company's page given a search query. +export async function findAndScrapeCompany(query: string): Promise { + const doc = await client.scrape(query, { + formats: ['markdown'], + }); + return doc.markdown ?? ''; +} + +// Scrape a documentation page with all available output formats. +export async function scrapeDocPage(url: string) { + const doc = await client.scrape(url, { + formats: ['markdown', 'html', 'links', 'screenshot'], + onlyMainContent: false, + }); + return { + markdown: doc.markdown, + html: doc.html, + links: doc.links, + screenshot: doc.screenshot, + }; +} diff --git a/examples/workbench/native-data-fetching/README.md b/examples/workbench/native-data-fetching/README.md new file mode 100644 index 0000000..f3a9a95 --- /dev/null +++ b/examples/workbench/native-data-fetching/README.md @@ -0,0 +1,54 @@ +# native-data-fetching eval + +Eval suite for +[`expo/skills/native-data-fetching`](https://github.com/expo/skills) — +prescribes Expo networking conventions: prefer `fetch` over axios, use +`SecureStore` for tokens, keep secrets out of `EXPO_PUBLIC_` env vars, always +check `response.ok`, and use `AbortController` for request cancellation. + +## Cases + +### `review-client` — axios, missing response.ok, insecure token storage + +Sample: `workspace/api/client.ts` + +| Line | Violation | Rule | +|---|---|---| +| 1 | `import axios from 'axios'` — axios used instead of native fetch | Preferences: "Avoid axios, prefer expo/fetch" | +| 12 | `axios.get(...)` — axios call in production fetch function | Preferences: "Avoid axios, prefer expo/fetch" | +| 18 | `response.json()` called without prior `response.ok` check | Common Mistakes: "Check response status" | +| 35 | `AsyncStorage.setItem('auth_token', token)` — token stored insecurely | Common Mistakes: "Use SecureStore for sensitive data" | + +### `review-dashboard` — exposed secret, no AbortController, axios again + +Sample: `workspace/screens/DashboardScreen.tsx` + +| Line | Violation | Rule | +|---|---|---| +| 5 | `EXPO_PUBLIC_STRIPE_SECRET_KEY` — secret embedded in client bundle | Env Variables: "Never put secrets in EXPO_PUBLIC_ variables" | +| 17–19 | `fetch(...)` in `useEffect` with no `AbortController` cleanup | Section 7: "Cancel on unmount" | +| 3 | `import axios from 'axios'` — axios used instead of fetch | Preferences: "Avoid axios, prefer expo/fetch" | +| 23 | `axios.get(...)` — axios call inside useEffect | Preferences: "Avoid axios, prefer expo/fetch" | + +## Vendored snapshot + +The skill normally references `references/expo-router-loaders.md` relative to +the workspace root. For deterministic eval we vendor a snapshot at +`references/expo-router-loaders.md`. The `SKILL.md` copy is verbatim from +upstream (no local-path tweak needed — the skill already uses relative local +paths, not WebFetch URLs). + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4.6` +- `openrouter/openai/gpt-5-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/native-data-fetching/analysis.md b/examples/workbench/native-data-fetching/analysis.md new file mode 100644 index 0000000..a657d75 --- /dev/null +++ b/examples/workbench/native-data-fetching/analysis.md @@ -0,0 +1,18 @@ +--- +skill: expo/skills/native-data-fetching +status: success +classification: code-reviewer +baseline_rule_coverage: 1.00 +final_rule_coverage: 1.00 +modifications_tried: 0 +total_cost_usd: 1.00 +--- + +# Auto-pilot run for `expo/skills/native-data-fetching` + +- Skill fetched from `plugins/expo/skills/native-data-fetching/SKILL.md` in the `expo/skills` repo (path differs from the simple `skills//SKILL.md` template — the repo uses a `plugins/expo/skills/` prefix). +- Classified as **code-reviewer**: prescribes Expo networking conventions (fetch over axios, SecureStore for tokens, keep secrets out of `EXPO_PUBLIC_` env vars, always check `response.ok`, use AbortController for cancellation) and the agent reviews code files against these rules. +- Seeded 2 TypeScript/TSX files with 3 violations each (6 total): `api/client.ts` (axios import, missing `response.ok` check, `AsyncStorage` token storage) and `screens/DashboardScreen.tsx` (secret in `EXPO_PUBLIC_`, no `AbortController`, axios usage). +- The skill already uses local `references/` paths rather than WebFetch URLs — no local-path tweak needed in the vendored copy. +- Baseline: 18/18 trials passed across 3 models × 2 cases × 3 trials = **rule-coverage 1.00**. No modifications required. +- Exiting clean per "exit clean on already-good skill" pattern from lessons.md. diff --git a/examples/workbench/native-data-fetching/checks/_grader-utils.mjs b/examples/workbench/native-data-fetching/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/native-data-fetching/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/native-data-fetching/checks/grade-client-findings.mjs b/examples/workbench/native-data-fetching/checks/grade-client-findings.mjs new file mode 100644 index 0000000..c3c6ca9 --- /dev/null +++ b/examples/workbench/native-data-fetching/checks/grade-client-findings.mjs @@ -0,0 +1,36 @@ +// Grader for review-client case. +// Checks that findings.txt correctly identifies violations in api/client.ts: +// V1 (axios-usage) line 1 — import axios / line 12 — axios.get call +// V2 (missing-response-ok) line 18 — response.json() without response.ok +// V3 (asyncstorage-token) line 35 — AsyncStorage.setItem for auth token + +import { join } from 'node:path'; +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK; +const findingsPath = join(WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'api/client.ts', + expected: [ + { + id: 'axios-usage', + // Agent may cite either the import (line 1) or the call (line 12) + lines: [...looseRange(1, 8), ...looseRange(12, 8)], + keywords: [/axios/i], + }, + { + id: 'missing-response-ok', + // Line 18: `const data = await response.json()` without prior response.ok check + lines: looseRange(18, 8), + keywords: [/response\.ok/i, /\.ok\b/i, /status.*check/i, /check.*status/i, /error.*handl/i, /missing.*check/i], + }, + { + id: 'asyncstorage-token', + // Line 35: `await AsyncStorage.setItem('auth_token', token)` + lines: looseRange(35, 8), + keywords: [fuzzyKeyword('AsyncStorage'), tolerantKeyword('SecureStore'), /insecure/i, /secure.*store/i, /not.*secure/i], + }, + ], +}); diff --git a/examples/workbench/native-data-fetching/checks/grade-dashboard-findings.mjs b/examples/workbench/native-data-fetching/checks/grade-dashboard-findings.mjs new file mode 100644 index 0000000..796d60d --- /dev/null +++ b/examples/workbench/native-data-fetching/checks/grade-dashboard-findings.mjs @@ -0,0 +1,36 @@ +// Grader for review-dashboard case. +// Checks that findings.txt correctly identifies violations in screens/DashboardScreen.tsx: +// V4 (expo-public-secret) line 5 — EXPO_PUBLIC_STRIPE_SECRET_KEY exposes a secret +// V5 (no-abort-controller) line 18 — fetch in useEffect with no AbortController cleanup +// V6 (axios-usage) line 3 — import axios / line 23 — axios.get call + +import { join } from 'node:path'; +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK; +const findingsPath = join(WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'screens/DashboardScreen.tsx', + expected: [ + { + id: 'expo-public-secret', + // Line 5: `const PAYMENT_KEY = process.env.EXPO_PUBLIC_STRIPE_SECRET_KEY` + lines: looseRange(5, 8), + keywords: [/EXPO_PUBLIC_/i, /secret/i, /stripe/i, /exposed/i, /visible/i, /client.*bundle/i, /bundle/i], + }, + { + id: 'no-abort-controller', + // Lines 17-19: fetch without AbortController in useEffect (no cleanup) + lines: looseRange(18, 8), + keywords: [/AbortController/i, /abort/i, /cancel/i, /cleanup/i, /unmount/i, /memory.*leak/i, /leak/i], + }, + { + id: 'axios-usage', + // Agent may cite import (line 3) or the call (line 23) + lines: [...looseRange(3, 8), ...looseRange(23, 8)], + keywords: [/axios/i], + }, + ], +}); diff --git a/examples/workbench/native-data-fetching/references/expo-router-loaders.md b/examples/workbench/native-data-fetching/references/expo-router-loaders.md new file mode 100644 index 0000000..ca3942c --- /dev/null +++ b/examples/workbench/native-data-fetching/references/expo-router-loaders.md @@ -0,0 +1,341 @@ +# Expo Router Data Loaders + +Route-level data loading for web apps using Expo SDK 55+. Loaders are async functions exported from route files that load data before the route renders, following the Remix/React Router loader model. + +**Dual execution model:** + +- **Initial page load (SSR):** The loader runs server-side. Its return value is serialized as JSON and embedded in the HTML response. +- **Client-side navigation:** The browser fetches the loader data from the server via HTTP. The route renders once the data arrives. + +You write one function and the framework manages when and how it executes. + +## Configuration + +**Requirements:** Expo SDK 55+, web output mode (`npx expo serve` or `npx expo export --platform web`) set in `app.json` or `app.config.js`. + +**Server rendering:** + +```json +{ + "expo": { + "web": { + "output": "server" + }, + "plugins": [ + ["expo-router", { + "unstable_useServerDataLoaders": true, + "unstable_useServerRendering": true + }] + ] + } +} +``` + +**Static/SSG:** + +```json +{ + "expo": { + "web": { + "output": "static" + }, + "plugins": [ + ["expo-router", { + "unstable_useServerDataLoaders": true + }] + ] + } +} +``` + +| | `"server"` | `"static"` | +|---|-----------|------------| +| `unstable_useServerDataLoaders` | Required | Required | +| `unstable_useServerRendering` | Required | Not required | +| Loader runs on | Live server (every request) | Build time (static generation) | +| `request` object | Full access (headers, cookies) | Not available | +| Hosting | Node.js server (EAS Hosting) | Any static host (Netlify, Vercel, S3) | + +## Imports + +Loaders use two packages: + +- **`expo-router`** — `useLoaderData` hook +- **`expo-server`** — `LoaderFunction` type, `StatusError`, `setResponseHeaders`. Always available (dependency of `expo-router`), no install needed. + +## Basic Loader + +For loaders without params, a plain async function works: + +```tsx +// app/posts/index.tsx +import { Suspense } from "react"; +import { useLoaderData } from "expo-router"; +import { ActivityIndicator, View, Text } from "react-native"; + +export async function loader() { + const response = await fetch("https://api.example.com/posts"); + const posts = await response.json(); + return { posts }; +} + +function PostList() { + const { posts } = useLoaderData(); + + return ( + + {posts.map((post) => ( + {post.title} + ))} + + ); +} + +export default function Posts() { + return ( + }> + + + ); +} +``` + +`useLoaderData` is typed via `typeof loader` — the generic parameter infers the return type. + +## Dynamic Routes + +For loaders with params, use the `LoaderFunction` type from `expo-server`. The first argument is the request (an immutable `Request`-like object, or `undefined` in static mode). The second is `params` (`Record`), which contains **path parameters only**. Access individual params with a cast like `params.id as string`. For query parameters, use `new URL(request.url).searchParams`: + +```tsx +// app/posts/[id].tsx +import { Suspense } from "react"; +import { useLoaderData } from "expo-router"; +import { StatusError, type LoaderFunction } from "expo-server"; +import { ActivityIndicator, View, Text } from "react-native"; + +type Post = { + id: number; + title: string; + body: string; +}; + +export const loader: LoaderFunction<{ post: Post }> = async ( + request, + params, +) => { + const id = params.id as string; + const response = await fetch(`https://api.example.com/posts/${id}`); + + if (!response.ok) { + throw new StatusError(404, `Post ${id} not found`); + } + + const post: Post = await response.json(); + return { post }; +}; + +function PostContent() { + const { post } = useLoaderData(); + + return ( + + {post.title} + {post.body} + + ); +} + +export default function PostDetail() { + return ( + }> + + + ); +} +``` + +Catch-all routes access `params.slug` the same way: + +```tsx +// app/docs/[...slug].tsx +import { type LoaderFunction } from "expo-server"; + +type Doc = { title: string; content: string }; + +export const loader: LoaderFunction<{ doc: Doc }> = async (request, params) => { + const slug = params.slug as string[]; + const path = slug.join("/"); + const doc = await fetchDoc(path); + return { doc }; +}; +``` + +Query parameters are available via the `request` object (server output mode only): + +```tsx +// app/search.tsx +import { type LoaderFunction } from "expo-server"; + +export const loader: LoaderFunction<{ results: any[]; query: string }> = async (request) => { + // Assuming request.url is `/search?q=expo&page=2` + const url = new URL(request!.url); + const query = url.searchParams.get("q") ?? ""; + const page = Number(url.searchParams.get("page") ?? "1"); + + const results = await fetchSearchResults(query, page); + return { results, query }; +}; +``` + +## Server-Side Secrets & Request Access + +Loaders run on the server, so you can access secrets and server-only resources directly: + +```tsx +// app/dashboard.tsx +import { type LoaderFunction } from "expo-server"; + +export const loader: LoaderFunction<{ balance: any; isAuthenticated: boolean }> = async ( + request, + params, +) => { + const data = await fetch("https://api.stripe.com/v1/balance", { + headers: { + Authorization: `Bearer ${process.env.STRIPE_SECRET_KEY}`, + }, + }); + + const sessionToken = request?.headers.get("cookie")?.match(/session=([^;]+)/)?.[1]; + + const balance = await data.json(); + return { balance, isAuthenticated: !!sessionToken }; +}; +``` + +The `request` object is available in server output mode. In static output mode, `request` is always `undefined`. + +## Response Utilities + +### Setting Response Headers + +```tsx +// app/products.tsx +import { setResponseHeaders } from "expo-server"; + +export async function loader() { + setResponseHeaders({ + "Cache-Control": "public, max-age=300", + }); + + const products = await fetchProducts(); + return { products }; +} +``` + +### Throwing HTTP Errors + +```tsx +// app/products/[id].tsx +import { StatusError, type LoaderFunction } from "expo-server"; + +export const loader: LoaderFunction<{ product: Product }> = async (request, params) => { + const id = params.id as string; + const product = await fetchProduct(id); + + if (!product) { + throw new StatusError(404, "Product not found"); + } + + return { product }; +}; +``` + +## Suspense & Error Boundaries + +### Loading States with Suspense + +`useLoaderData()` suspends during client-side navigation. Push it into a child component and wrap with ``: + +```tsx +// app/posts/index.tsx +import { Suspense } from "react"; +import { useLoaderData } from "expo-router"; +import { ActivityIndicator, View, Text } from "react-native"; + +export async function loader() { + const response = await fetch("https://api.example.com/posts"); + return { posts: await response.json() }; +} + +function PostList() { + const { posts } = useLoaderData(); + + return ( + + {posts.map((post) => ( + {post.title} + ))} + + ); +} + +export default function Posts() { + return ( + + + + } + > + + + ); +} +``` + +The `` boundary must be above the component calling `useLoaderData()`. On initial page load the data is already in the HTML, suspension only occurs during client-side navigation. + +### Error Boundaries + +```tsx +// app/posts/[id].tsx +export function ErrorBoundary({ error }: { error: Error }) { + return ( + + Error: {error.message} + + ); +} +``` + +When a loader throws (including `StatusError`), the nearest `ErrorBoundary` catches it. + +## Static vs Server Rendering + +| | Server (`"server"`) | Static (`"static"`) | +|---|---|---| +| **When loader runs** | Every request (live) | At build time (`npx expo export`) | +| **Data freshness** | Fresh on initial server request | Stale until next build | +| **`request` object** | Full access | Not available | +| **Hosting** | Node.js server (EAS Hosting) | Any static host | +| **Use case** | Personalized/dynamic content | Marketing pages, blogs, docs | + +**Choose server** when data changes frequently or content is personalized (cookies, auth, headers). + +**Choose static** when content is the same for all users and changes infrequently. + +## Best Practices + +- Loaders are web-only; use client-side fetching (React Query, fetch) for native +- Loaders cannot be used in `_layout` files — only in route files +- Use `LoaderFunction` from `expo-server` to type loaders that use params +- The request object is immutable — use optional chaining (`request?.headers`) as it may be `undefined` in static mode +- Return only JSON-serializable values (no `Date`, `Map`, `Set`, class instances, functions) +- Use non-prefixed `process.env` vars for secrets in loaders, not `EXPO_PUBLIC_` (which is embedded in the client bundle) +- Use `StatusError` from `expo-server` for HTTP error responses +- Use `setResponseHeaders` from `expo-server` to set headers +- Export `ErrorBoundary` from route files to handle loader failures gracefully +- Validate and sanitize user input (params, query strings) before using in database queries or API calls +- Handle errors gracefully with try/catch; log server-side for debugging +- Loader data is currently cached for the session. This is a known limitation that will be lifted in a future release diff --git a/examples/workbench/native-data-fetching/references/native-data-fetching/SKILL.md b/examples/workbench/native-data-fetching/references/native-data-fetching/SKILL.md new file mode 100644 index 0000000..d59cb38 --- /dev/null +++ b/examples/workbench/native-data-fetching/references/native-data-fetching/SKILL.md @@ -0,0 +1,507 @@ +--- +name: native-data-fetching +description: Use when implementing or debugging ANY network request, API call, or data fetching. Covers fetch API, React Query, SWR, error handling, caching, offline support, and Expo Router data loaders (`useLoaderData`). +version: 1.0.0 +license: MIT +--- + +# Expo Networking + +**You MUST use this skill for ANY networking work including API requests, data fetching, caching, or network debugging.** + +## References + +Consult these resources as needed: + +``` +references/ + expo-router-loaders.md Route-level data loading with Expo Router loaders (web, SDK 55+) +``` + +## When to Use + +Use this skill when: + +- Implementing API requests +- Setting up data fetching (React Query, SWR) +- Using Expo Router data loaders (`useLoaderData`, web SDK 55+) +- Debugging network failures +- Implementing caching strategies +- Handling offline scenarios +- Authentication/token management +- Configuring API URLs and environment variables + +## Preferences + +- Avoid axios, prefer expo/fetch + +## Common Issues & Solutions + +### 1. Basic Fetch Usage + +**Simple GET request**: + +```tsx +const fetchUser = async (userId: string) => { + const response = await fetch(`https://api.example.com/users/${userId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return response.json(); +}; +``` + +**POST request with body**: + +```tsx +const createUser = async (userData: UserData) => { + const response = await fetch("https://api.example.com/users", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(userData), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.message); + } + + return response.json(); +}; +``` + +--- + +### 2. React Query (TanStack Query) + +**Setup**: + +```tsx +// app/_layout.tsx +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 1000 * 60 * 5, // 5 minutes + retry: 2, + }, + }, +}); + +export default function RootLayout() { + return ( + + + + ); +} +``` + +**Fetching data**: + +```tsx +import { useQuery } from "@tanstack/react-query"; + +function UserProfile({ userId }: { userId: string }) { + const { data, isLoading, error, refetch } = useQuery({ + queryKey: ["user", userId], + queryFn: () => fetchUser(userId), + }); + + if (isLoading) return ; + if (error) return ; + + return ; +} +``` + +**Mutations**: + +```tsx +import { useMutation, useQueryClient } from "@tanstack/react-query"; + +function CreateUserForm() { + const queryClient = useQueryClient(); + + const mutation = useMutation({ + mutationFn: createUser, + onSuccess: () => { + // Invalidate and refetch + queryClient.invalidateQueries({ queryKey: ["users"] }); + }, + }); + + const handleSubmit = (data: UserData) => { + mutation.mutate(data); + }; + + return
; +} +``` + +--- + +### 3. Error Handling + +**Comprehensive error handling**: + +```tsx +class ApiError extends Error { + constructor(message: string, public status: number, public code?: string) { + super(message); + this.name = "ApiError"; + } +} + +const fetchWithErrorHandling = async (url: string, options?: RequestInit) => { + try { + const response = await fetch(url, options); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + throw new ApiError( + error.message || "Request failed", + response.status, + error.code + ); + } + + return response.json(); + } catch (error) { + if (error instanceof ApiError) { + throw error; + } + // Network error (no internet, timeout, etc.) + throw new ApiError("Network error", 0, "NETWORK_ERROR"); + } +}; +``` + +**Retry logic**: + +```tsx +const fetchWithRetry = async ( + url: string, + options?: RequestInit, + retries = 3 +) => { + for (let i = 0; i < retries; i++) { + try { + return await fetchWithErrorHandling(url, options); + } catch (error) { + if (i === retries - 1) throw error; + // Exponential backoff + await new Promise((r) => setTimeout(r, Math.pow(2, i) * 1000)); + } + } +}; +``` + +--- + +### 4. Authentication + +**Token management**: + +```tsx +import * as SecureStore from "expo-secure-store"; + +const TOKEN_KEY = "auth_token"; + +export const auth = { + getToken: () => SecureStore.getItemAsync(TOKEN_KEY), + setToken: (token: string) => SecureStore.setItemAsync(TOKEN_KEY, token), + removeToken: () => SecureStore.deleteItemAsync(TOKEN_KEY), +}; + +// Authenticated fetch wrapper +const authFetch = async (url: string, options: RequestInit = {}) => { + const token = await auth.getToken(); + + return fetch(url, { + ...options, + headers: { + ...options.headers, + Authorization: token ? `Bearer ${token}` : "", + }, + }); +}; +``` + +**Token refresh**: + +```tsx +let isRefreshing = false; +let refreshPromise: Promise | null = null; + +const getValidToken = async (): Promise => { + const token = await auth.getToken(); + + if (!token || isTokenExpired(token)) { + if (!isRefreshing) { + isRefreshing = true; + refreshPromise = refreshToken().finally(() => { + isRefreshing = false; + refreshPromise = null; + }); + } + return refreshPromise!; + } + + return token; +}; +``` + +--- + +### 5. Offline Support + +**Check network status**: + +```tsx +import NetInfo from "@react-native-community/netinfo"; + +// Hook for network status +function useNetworkStatus() { + const [isOnline, setIsOnline] = useState(true); + + useEffect(() => { + return NetInfo.addEventListener((state) => { + setIsOnline(state.isConnected ?? true); + }); + }, []); + + return isOnline; +} +``` + +**Offline-first with React Query**: + +```tsx +import { onlineManager } from "@tanstack/react-query"; +import NetInfo from "@react-native-community/netinfo"; + +// Sync React Query with network status +onlineManager.setEventListener((setOnline) => { + return NetInfo.addEventListener((state) => { + setOnline(state.isConnected ?? true); + }); +}); + +// Queries will pause when offline and resume when online +``` + +--- + +### 6. Environment Variables + +**Using environment variables for API configuration**: + +Expo supports environment variables with the `EXPO_PUBLIC_` prefix. These are inlined at build time and available in your JavaScript code. + +```tsx +// .env +EXPO_PUBLIC_API_URL=https://api.example.com +EXPO_PUBLIC_API_VERSION=v1 + +// Usage in code +const API_URL = process.env.EXPO_PUBLIC_API_URL; + +const fetchUsers = async () => { + const response = await fetch(`${API_URL}/users`); + return response.json(); +}; +``` + +**Environment-specific configuration**: + +```tsx +// .env.development +EXPO_PUBLIC_API_URL=http://localhost:3000 + +// .env.production +EXPO_PUBLIC_API_URL=https://api.production.com +``` + +**Creating an API client with environment config**: + +```tsx +// api/client.ts +const BASE_URL = process.env.EXPO_PUBLIC_API_URL; + +if (!BASE_URL) { + throw new Error("EXPO_PUBLIC_API_URL is not defined"); +} + +export const apiClient = { + get: async (path: string): Promise => { + const response = await fetch(`${BASE_URL}${path}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.json(); + }, + + post: async (path: string, body: unknown): Promise => { + const response = await fetch(`${BASE_URL}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.json(); + }, +}; +``` + +**Important notes**: + +- Only variables prefixed with `EXPO_PUBLIC_` are exposed to the client bundle +- Never put secrets (API keys with write access, database passwords) in `EXPO_PUBLIC_` variables—they're visible in the built app +- Environment variables are inlined at **build time**, not runtime +- Restart the dev server after changing `.env` files +- For server-side secrets in API routes, use variables without the `EXPO_PUBLIC_` prefix + +**TypeScript support**: + +```tsx +// types/env.d.ts +declare global { + namespace NodeJS { + interface ProcessEnv { + EXPO_PUBLIC_API_URL: string; + EXPO_PUBLIC_API_VERSION?: string; + } + } +} + +export {}; +``` + +--- + +### 7. Request Cancellation + +**Cancel on unmount**: + +```tsx +useEffect(() => { + const controller = new AbortController(); + + fetch(url, { signal: controller.signal }) + .then((response) => response.json()) + .then(setData) + .catch((error) => { + if (error.name !== "AbortError") { + setError(error); + } + }); + + return () => controller.abort(); +}, [url]); +``` + +**With React Query** (automatic): + +```tsx +// React Query automatically cancels requests when queries are invalidated +// or components unmount +``` + +--- + +## Decision Tree + +``` +User asks about networking + |-- Route-level data loading (web, SDK 55+)? + | \-- Expo Router loaders — see references/expo-router-loaders.md + | + |-- Basic fetch? + | \-- Use fetch API with error handling + | + |-- Need caching/state management? + | |-- Complex app -> React Query (TanStack Query) + | \-- Simpler needs -> SWR or custom hooks + | + |-- Authentication? + | |-- Token storage -> expo-secure-store + | \-- Token refresh -> Implement refresh flow + | + |-- Error handling? + | |-- Network errors -> Check connectivity first + | |-- HTTP errors -> Parse response, throw typed errors + | \-- Retries -> Exponential backoff + | + |-- Offline support? + | |-- Check status -> NetInfo + | \-- Queue requests -> React Query persistence + | + |-- Environment/API config? + | |-- Client-side URLs -> EXPO_PUBLIC_ prefix in .env + | |-- Server secrets -> Non-prefixed env vars (API routes only) + | \-- Multiple environments -> .env.development, .env.production + | + \-- Performance? + |-- Caching -> React Query with staleTime + |-- Deduplication -> React Query handles this + \-- Cancellation -> AbortController or React Query +``` + +## Common Mistakes + +**Wrong: No error handling** + +```tsx +const data = await fetch(url).then((r) => r.json()); +``` + +**Right: Check response status** + +```tsx +const response = await fetch(url); +if (!response.ok) throw new Error(`HTTP ${response.status}`); +const data = await response.json(); +``` + +**Wrong: Storing tokens in AsyncStorage** + +```tsx +await AsyncStorage.setItem("token", token); // Not secure! +``` + +**Right: Use SecureStore for sensitive data** + +```tsx +await SecureStore.setItemAsync("token", token); +``` + +## Example Invocations + +User: "How do I make API calls in React Native?" +-> Use fetch, wrap with error handling + +User: "Should I use React Query or SWR?" +-> React Query for complex apps, SWR for simpler needs + +User: "My app needs to work offline" +-> Use NetInfo for status, React Query persistence for caching + +User: "How do I handle authentication tokens?" +-> Store in expo-secure-store, implement refresh flow + +User: "API calls are slow" +-> Check caching strategy, use React Query staleTime + +User: "How do I configure different API URLs for dev and prod?" +-> Use EXPO*PUBLIC* env vars with .env.development and .env.production files + +User: "Where should I put my API key?" +-> Client-safe keys: EXPO*PUBLIC* in .env. Secret keys: non-prefixed env vars in API routes only + +User: "How do I load data for a page in Expo Router?" +-> See references/expo-router-loaders.md for route-level loaders (web, SDK 55+). For native, use React Query or fetch. diff --git a/examples/workbench/native-data-fetching/suite.yml b/examples/workbench/native-data-fetching/suite.yml new file mode 100644 index 0000000..894da4f --- /dev/null +++ b/examples/workbench/native-data-fetching/suite.yml @@ -0,0 +1,42 @@ +name: native-data-fetching-eval +references: ./references +workspace: ./workspace +models: + - openrouter/anthropic/claude-sonnet-4.6 + - openrouter/openai/gpt-5-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + Keep task outputs at the top level of /work unless the user asks otherwise. +cases: + - name: review-client + task: | + A teammate submitted a pull request adding `api/client.ts`. Review this file for + networking and data-fetching issues using the Expo skill at + `references/native-data-fetching/SKILL.md`. + + Write all findings to `findings.txt`. Each line must identify the file, line number, + and issue, for example: + api/client.ts:12 — description of the problem and which rule it breaks + + Be thorough — check every function in the file. + graders: + - name: client-findings + command: node $CASE/checks/grade-client-findings.mjs + + - name: review-dashboard + task: | + A teammate submitted a pull request adding `screens/DashboardScreen.tsx`. Review this + file for networking and data-fetching issues using the Expo skill at + `references/native-data-fetching/SKILL.md`. + + Write all findings to `findings.txt`. Each line must identify the file, line number, + and issue, for example: + screens/DashboardScreen.tsx:5 — description of the problem and which rule it breaks + + Be thorough — check imports, environment variables, useEffect hooks, and every fetch call. + graders: + - name: dashboard-findings + command: node $CASE/checks/grade-dashboard-findings.mjs diff --git a/examples/workbench/native-data-fetching/workspace/api/client.ts b/examples/workbench/native-data-fetching/workspace/api/client.ts new file mode 100644 index 0000000..596c33e --- /dev/null +++ b/examples/workbench/native-data-fetching/workspace/api/client.ts @@ -0,0 +1,40 @@ +import axios from 'axios'; +import AsyncStorage from '@react-native-async-storage/async-storage'; + +const BASE_URL = process.env.EXPO_PUBLIC_API_URL || 'https://api.example.com'; + +interface UserData { + name: string; + email: string; +} + +export const getUser = async (userId: string) => { + const response = await axios.get(`${BASE_URL}/users/${userId}`); + return response.data; +}; + +export const getUsers = async () => { + const response = await fetch(`${BASE_URL}/users`); + const data = await response.json(); + return data; +}; + +export const createUser = async (userData: UserData) => { + const response = await fetch(`${BASE_URL}/users`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(userData), + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return response.json(); +}; + +export const saveAuthToken = async (token: string) => { + await AsyncStorage.setItem('auth_token', token); +}; + +export const getAuthToken = async () => { + return AsyncStorage.getItem('auth_token'); +}; diff --git a/examples/workbench/native-data-fetching/workspace/screens/DashboardScreen.tsx b/examples/workbench/native-data-fetching/workspace/screens/DashboardScreen.tsx new file mode 100644 index 0000000..e23ea32 --- /dev/null +++ b/examples/workbench/native-data-fetching/workspace/screens/DashboardScreen.tsx @@ -0,0 +1,38 @@ +import React, { useEffect, useState } from 'react'; +import { View, Text, FlatList } from 'react-native'; +import axios from 'axios'; + +const PAYMENT_KEY = process.env.EXPO_PUBLIC_STRIPE_SECRET_KEY; + +interface Post { + id: number; + title: string; +} + +export function DashboardScreen() { + const [profile, setProfile] = useState(null); + const [posts, setPosts] = useState([]); + + useEffect(() => { + fetch('https://api.example.com/profile') + .then((r) => r.json()) + .then(setProfile); + }, []); + + useEffect(() => { + axios.get('https://api.example.com/posts').then((res) => { + setPosts(res.data); + }); + }, []); + + return ( + + Dashboard + {item.title}} + keyExtractor={(item) => item.id.toString()} + /> + + ); +} diff --git a/examples/workbench/next-best-practices/README.md b/examples/workbench/next-best-practices/README.md new file mode 100644 index 0000000..e05784e --- /dev/null +++ b/examples/workbench/next-best-practices/README.md @@ -0,0 +1,53 @@ +# next-best-practices eval + +Eval suite for +[`vercel-labs/next-skills/next-best-practices`](https://github.com/vercel-labs/next-skills) — +a comprehensive code-reviewer skill covering Next.js 15+ patterns: RSC boundaries, +async APIs, image/font optimization, error handling, data patterns, and more. + +## Cases + +### `review-dashboard` — async patterns, data patterns, error handling, RSC boundaries, image + +Sample: `workspace/app/dashboard/page.tsx` + +| Line | Violation | Rule | +|---|---|---| +| 9 | Synchronous `params` access — must `await params` in Next.js 15+ | async-patterns | +| 12–13 | Sequential `await` fetches create a data waterfall — use `Promise.all` | data-patterns | +| 19 | `redirect()` called inside `try-catch` — swallows the navigation throw | error-handling | +| 31 | `Date` object passed as prop to a client component — not JSON-serializable | rsc-boundaries | +| 33 | Native `` tag used instead of `next/image` | image | + +### `review-herosection` — RSC boundaries, image, scripts + +Sample: `workspace/components/HeroSection.tsx` + +| Line | Violation | Rule | +|---|---|---| +| 6 | `async` client component — `'use client'` + `async function` is invalid | rsc-boundaries | +| 12 | `` without `sizes` prop — downloads largest image regardless of viewport | image | +| 17 | Missing `priority` prop on above-the-fold LCP hero image | image | +| 24 | Native ` + +// Good: Next.js Script component +import Script from 'next/script' + + +``` + +## Don't Put Script in Head + +`next/script` should not be placed inside `next/head`. It handles its own positioning. + +```tsx +// Bad: Script inside Head +import Head from 'next/head' +import Script from 'next/script' + + + + +// Good: Next.js component +import { GoogleAnalytics } from '@next/third-parties/google' + +export default function Layout({ children }) { + return ( + + {children} + + + ) +} +``` + +## Google Tag Manager + +```tsx +import { GoogleTagManager } from '@next/third-parties/google' + +export default function Layout({ children }) { + return ( + + + {children} + + ) +} +``` + +## Other Third-Party Scripts + +```tsx +// YouTube embed +import { YouTubeEmbed } from '@next/third-parties/google' + + + +// Google Maps +import { GoogleMapsEmbed } from '@next/third-parties/google' + + +``` + +## Quick Reference + +| Pattern | Issue | Fix | +|---------|-------|-----| +| ` +

{config.title}

+ + ) +} diff --git a/examples/workbench/next-upgrade/README.md b/examples/workbench/next-upgrade/README.md new file mode 100644 index 0000000..0bd4b2e --- /dev/null +++ b/examples/workbench/next-upgrade/README.md @@ -0,0 +1,50 @@ +# next-upgrade eval + +Eval suite for +[`vercel-labs/next-skills/next-upgrade`](https://github.com/vercel-labs/next-skills) — +upgrade Next.js to the latest version following official migration guides and codemods. + +## Cases + +### `upgrade-starter-app` — v14→v15 async Request API migration + +Sample: `workspace/starter-app/` (Next.js 14 project) + +| File | Line | Violation | Rule | +|---|---|---|---| +| `package.json` | 10 | `next` version is `14.2.5`, not v15 | Install Updates (Step 5) | +| `app/page.tsx` | 4–9 | `viewport` is inside `metadata` export instead of separate `viewport` export | Manual Review (Step 6) | +| `app/page.tsx` | 14 | `searchParams` type is synchronous `{ query?: string }` instead of `Promise<{ query?: string }>` | Async Request APIs (Step 4) | +| `app/[id]/page.tsx` | 4 | `params` type is synchronous `{ id: string }` instead of `Promise<{ id: string }>` | Async Request APIs (Step 4) | +| `app/api/route.ts` | 7 | `cookies()` called without `await` | Async Request APIs (Step 4) | +| `app/api/route.ts` | 8 | `headers()` called without `await` | Async Request APIs (Step 4) | + +## Graders + +Graders check the **modified workspace files** after the agent applies changes: + +- `grade-starter-package.mjs` — checks `package.json` next version updated to v15 +- `grade-starter-pages.mjs` — checks `app/page.tsx` (viewport export + async searchParams) and `app/[id]/page.tsx` (async params) +- `grade-starter-route.mjs` — checks `app/api/route.ts` for `await cookies()` and `await headers()` + +## Vendored snapshot + +The skill normally fetches upgrade guides from `docs.next.js.org` at runtime. +For deterministic eval we vendor a snapshot at +`references/next-upgrade/upgrade-guide.md` and tweak `SKILL.md` to read it +locally. Diff vs upstream is one line (Step 2 WebFetch → local file read). + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-5` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/next-upgrade/analysis.md b/examples/workbench/next-upgrade/analysis.md new file mode 100644 index 0000000..9a5357f --- /dev/null +++ b/examples/workbench/next-upgrade/analysis.md @@ -0,0 +1,48 @@ +--- +skill: vercel-labs/next-skills/next-upgrade +status: uplift-too-small +classification: code-reviewer +baseline_rule_coverage: 0.83 +final_rule_coverage: 0.76 +modifications_tried: 2 +total_cost_usd: 0.94 +--- + +# Auto-pilot run for `vercel-labs/next-skills/next-upgrade` + +- **Classification:** Initially classified as code-patterns (agent applies transforms), but + workspace permission issues (files read-only in Docker) made code-modification graders + unreliable. Reclassified as code-reviewer — agent reads files and writes findings.txt. + +- **Seed:** 1 case (`review-starter-app`) with 6 violations across 4 files: + `package.json` (v14 version), `app/page.tsx` (viewport in metadata + sync searchParams), + `app/[id]/page.tsx` (sync params), `app/api/route.ts` (sync cookies + headers). + +- **Grader calibration:** Initial graders had two problems: (1) violation COMMENTS in seed + files contained the exact patterns the graders checked (all code-modification graders + false-passed); (2) `pkg-version` grader used `looseRange(12)` but models write + `package.json:1` or `:2` for file-level version issues (off by 10+ lines). Fixed by + switching to `findings.txt` shape and using `range(1,25)` for package.json. Calibration + runs not counted in iteration budget. + +- **Baseline (after calibration):** 45/54 = 0.833 rule-coverage. Claude 3/3 perfect; + Gemini 2/3 perfect (1 miss: route-headers); GPT-4o-mini 2/3 had 5/6 (missed + page-searchparams). 1/3 GPT trial failed completely: model ran `npx next-upgrade` + (fabricated CLI), got error, wrote error to findings.txt. + +- **Iteration 1:** Added per-element grep checklist with bash commands to SKILL.md. + Coverage DROPPED to 0.685 — bash commands caused GPT-4o-mini to try executing them + rather than reading files, producing further confusion. Recipe for this skill type: + do NOT include bash commands in skill instructions. + +- **Iteration 2:** Replaced bash commands with pure BAD/GOOD code examples (Recipe D) + for async params/searchParams. Coverage was 0.759 — still below baseline. GPT-4o-mini + still inconsistent: 1 trial perfect, 2 trials near-zero (model tried CLI tools or + wrote very sparse findings). Gemini also had 1 trial miss searchParams/params. + +- **Root cause of uplift failure:** GPT-4o-mini (and occasionally Gemini) tries to run + `npx next-upgrade` or similar CLI tools before reviewing files. When the CLI fails, + the model either gives up or writes the error as findings. Neither skill modification + addressed this CLI-fixation behavior reliably. The BAD/GOOD examples helped some + Gemini trials but couldn't overcome GPT's architectural limitation on tool-use vs + code-review framing. diff --git a/examples/workbench/next-upgrade/checks/_grader-utils.mjs b/examples/workbench/next-upgrade/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/next-upgrade/checks/grade-id-page-findings.mjs b/examples/workbench/next-upgrade/checks/grade-id-page-findings.mjs new file mode 100644 index 0000000..143b5d3 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-id-page-findings.mjs @@ -0,0 +1,22 @@ +// Grader: checks findings.txt for v14→v15 violations in app/[id]/page.tsx. +// +// Violations: +// id-params — params type is synchronous / must be awaited (lines 4–6) + +import { gradeFindings, looseRange, fuzzyKeyword } from './_grader-utils.mjs'; +import { join } from 'node:path'; + +const findingsPath = join(process.env.WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'app/[id]/page.tsx', + expected: [ + { + id: 'id-params', + // Center on line 5 (midpoint of type at 4 and access at 6), ±8 tolerance + lines: looseRange(5), + keywords: [fuzzyKeyword('params'), /async|Promise/i], + }, + ], +}); diff --git a/examples/workbench/next-upgrade/checks/grade-package-findings.mjs b/examples/workbench/next-upgrade/checks/grade-package-findings.mjs new file mode 100644 index 0000000..6a7d0a3 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-package-findings.mjs @@ -0,0 +1,27 @@ +// Grader: checks findings.txt for v14→v15 violations in package.json. +// +// Violations: +// pkg-version — next version is 14.x, should be upgraded to v15 +// +// Line-range note: models consistently report package.json version issues at +// line 1 or 2 (top of file) rather than the dependency line (~12). This is a +// common drift for file-level issues. We accept any line in the file (1–25). +// See lessons.md § G1. + +import { gradeFindings, range, tolerantKeyword } from './_grader-utils.mjs'; +import { join } from 'node:path'; + +const findingsPath = join(process.env.WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'package.json', + expected: [ + { + id: 'pkg-version', + // Accept any line in the file — models report this at line 1 or 2 + lines: range(1, 25), + keywords: [tolerantKeyword('next'), /14|15|version|upgrade/i], + }, + ], +}); diff --git a/examples/workbench/next-upgrade/checks/grade-page-findings.mjs b/examples/workbench/next-upgrade/checks/grade-page-findings.mjs new file mode 100644 index 0000000..a0e926c --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-page-findings.mjs @@ -0,0 +1,27 @@ +// Grader: checks findings.txt for v14→v15 violations in app/page.tsx. +// +// Violations: +// page-viewport — viewport is inside `metadata` export (line 6) +// page-searchparams — searchParams prop type is synchronous (line 15) + +import { gradeFindings, looseRange, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; +import { join } from 'node:path'; + +const findingsPath = join(process.env.WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'app/page.tsx', + expected: [ + { + id: 'page-viewport', + lines: looseRange(6), + keywords: [fuzzyKeyword('viewport')], + }, + { + id: 'page-searchparams', + lines: looseRange(15), + keywords: [fuzzyKeyword('searchParams'), /async|Promise/i], + }, + ], +}); diff --git a/examples/workbench/next-upgrade/checks/grade-route-findings.mjs b/examples/workbench/next-upgrade/checks/grade-route-findings.mjs new file mode 100644 index 0000000..7b656f3 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-route-findings.mjs @@ -0,0 +1,27 @@ +// Grader: checks findings.txt for v14→v15 violations in app/api/route.ts. +// +// Violations: +// route-cookies — cookies() is synchronous (line 5) +// route-headers — headers() is synchronous (line 6) + +import { gradeFindings, looseRange, fuzzyKeyword } from './_grader-utils.mjs'; +import { join } from 'node:path'; + +const findingsPath = join(process.env.WORK, 'findings.txt'); + +gradeFindings({ + findingsPath, + file: 'app/api/route.ts', + expected: [ + { + id: 'route-cookies', + lines: looseRange(5), + keywords: [fuzzyKeyword('cookies')], + }, + { + id: 'route-headers', + lines: looseRange(6), + keywords: [fuzzyKeyword('headers')], + }, + ], +}); diff --git a/examples/workbench/next-upgrade/checks/grade-starter-package.mjs b/examples/workbench/next-upgrade/checks/grade-starter-package.mjs new file mode 100644 index 0000000..9bde808 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-starter-package.mjs @@ -0,0 +1,48 @@ +// Grader: checks that package.json was updated to next v15. +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const work = process.env.WORK; +const pkgPath = join(work, 'starter-app', 'package.json'); + +const violations = [ + { id: 'pkg-next-v15', label: 'next version updated to v15' }, +]; + +const found = new Set(); +const evidence = []; + +if (!existsSync(pkgPath)) { + evidence.push('FAIL: starter-app/package.json not found'); +} else { + let pkg; + try { + pkg = JSON.parse(readFileSync(pkgPath, 'utf-8')); + } catch { + evidence.push('FAIL: could not parse package.json'); + pkg = null; + } + + if (pkg) { + const nextVer = pkg.dependencies?.next ?? pkg.devDependencies?.next ?? ''; + // Accept: "^15", "~15", "15.x.x", "latest", "canary", ">14", ">=15" + if (/\b15\b|latest|canary|>14/.test(nextVer)) { + found.add('pkg-next-v15'); + evidence.push(`+ pkg-next-v15: next version = "${nextVer}"`); + } else { + evidence.push(`- pkg-next-v15: next version still "${nextVer}", expected v15`); + } + } +} + +const score = found.size / violations.length; +const pass = found.size === violations.length; + +console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${violations.length} package violations fixed`, + ...evidence, + ], +})); diff --git a/examples/workbench/next-upgrade/checks/grade-starter-pages.mjs b/examples/workbench/next-upgrade/checks/grade-starter-pages.mjs new file mode 100644 index 0000000..d39eaac --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-starter-pages.mjs @@ -0,0 +1,72 @@ +// Grader: checks async API fixes in app/page.tsx and app/[id]/page.tsx. +// +// Violations: +// page-viewport — viewport must be a separate export (not inside metadata) +// page-searchparams — searchParams must be async (Promise<> type or awaited) +// id-page-params — params must be async (Promise<> type or awaited) + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const work = process.env.WORK; + +function readFile(relPath) { + const abs = join(work, 'starter-app', relPath); + if (!existsSync(abs)) return null; + return readFileSync(abs, 'utf-8'); +} + +const found = new Set(); +const evidence = []; +const total = 3; + +// --- page.tsx --- +const pageSrc = readFile('app/page.tsx'); +if (!pageSrc) { + evidence.push('FAIL: app/page.tsx not found'); +} else { + // Violation 1: viewport should be a separate export const viewport + // Accept: `export const viewport` or `export function generateViewport` + if (/export\s+(const\s+viewport|function\s+generateViewport)/i.test(pageSrc)) { + found.add('page-viewport'); + evidence.push('+ page-viewport: separate viewport export found'); + } else { + evidence.push('- page-viewport: no separate viewport export; viewport still inside metadata or missing'); + } + + // Violation 2: searchParams must be async + // Accept: `Promise<` in type annotation near searchParams, or `await searchParams` + if (/Promise\s*<[^>]*>\s*[,}]/.test(pageSrc) || /await\s+searchParams/.test(pageSrc)) { + found.add('page-searchparams'); + evidence.push('+ page-searchparams: searchParams treated as async'); + } else { + evidence.push('- page-searchparams: searchParams still synchronous (no Promise<> type or await)'); + } +} + +// --- app/[id]/page.tsx --- +const idPageSrc = readFile('app/[id]/page.tsx'); +if (!idPageSrc) { + evidence.push('FAIL: app/[id]/page.tsx not found'); +} else { + // Violation 3: params must be async + // Accept: `Promise<` in type annotation near params, or `await params` + if (/Promise\s*<[^>]*>\s*[,}]/.test(idPageSrc) || /await\s+params/.test(idPageSrc)) { + found.add('id-page-params'); + evidence.push('+ id-page-params: params treated as async'); + } else { + evidence.push('- id-page-params: params still synchronous (no Promise<> type or await)'); + } +} + +const score = found.size / total; +const pass = found.size === total; + +console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${total} page async-API violations fixed`, + ...evidence, + ], +})); diff --git a/examples/workbench/next-upgrade/checks/grade-starter-route.mjs b/examples/workbench/next-upgrade/checks/grade-starter-route.mjs new file mode 100644 index 0000000..32c1521 --- /dev/null +++ b/examples/workbench/next-upgrade/checks/grade-starter-route.mjs @@ -0,0 +1,60 @@ +// Grader: checks async API fixes in app/api/route.ts. +// +// Violations: +// route-cookies — cookies() must be awaited in v15 +// route-headers — headers() must be awaited in v15 + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const work = process.env.WORK; +const routePath = join(work, 'starter-app', 'app', 'api', 'route.ts'); + +const found = new Set(); +const evidence = []; +const total = 2; + +if (!existsSync(routePath)) { + evidence.push('FAIL: app/api/route.ts not found'); +} else { + const src = readFileSync(routePath, 'utf-8'); + + // Violation 1: cookies() must be awaited + // Accept: `await cookies()` anywhere in the file + if (/await\s+cookies\s*\(\s*\)/.test(src)) { + found.add('route-cookies'); + evidence.push('+ route-cookies: await cookies() found'); + } else { + // Also check if cookies is no longer used (agent removed it) — partial credit not given + // but note if it was removed entirely + if (!/cookies/.test(src)) { + evidence.push('- route-cookies: cookies() removed entirely (no await pattern found)'); + } else { + evidence.push('- route-cookies: cookies() still synchronous (no await)'); + } + } + + // Violation 2: headers() must be awaited + if (/await\s+headers\s*\(\s*\)/.test(src)) { + found.add('route-headers'); + evidence.push('+ route-headers: await headers() found'); + } else { + if (!/headers/.test(src)) { + evidence.push('- route-headers: headers() removed entirely (no await pattern found)'); + } else { + evidence.push('- route-headers: headers() still synchronous (no await)'); + } + } +} + +const score = found.size / total; +const pass = found.size === total; + +console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${total} route async-API violations fixed`, + ...evidence, + ], +})); diff --git a/examples/workbench/next-upgrade/references/next-upgrade/SKILL.md b/examples/workbench/next-upgrade/references/next-upgrade/SKILL.md new file mode 100644 index 0000000..4fc4f5e --- /dev/null +++ b/examples/workbench/next-upgrade/references/next-upgrade/SKILL.md @@ -0,0 +1,114 @@ +# Upgrade Next.js - SKILL.md + +**Name:** next-upgrade +**Description:** Upgrade Next.js to the latest version following official migration guides and codemods +**Argument Hint:** [target-version] + +## Overview + +This skill guides you through upgrading Next.js to the latest version by following official documentation and automated tools. + +## Key Steps + +1. **Identify Current Setup**: Examine `package.json` to determine your existing Next.js version and related dependencies like React and React DOM. + +2. **Access Upgrade Documentation**: Read the vendored upgrade guide at `/work/references/next-upgrade/upgrade-guide.md`, which contains version-specific migration paths and codemod information for v14, v15, and v16 upgrades. + +3. **Plan Migration Strategy**: For significant version jumps, perform incremental upgrades rather than jumping multiple versions at once. + +4. **Apply Automated Transforms**: Use Next.js codemods to handle breaking changes automatically: + - "Updates async Request APIs (v15)" + - "Migrates geo/ip properties (v15)" + - "Transforms dynamic imports (v15)" + +5. **Install Updates**: Upgrade Next.js alongside peer dependencies using npm. + +6. **Manual Review**: Consult upgrade guides for changes requiring manual intervention, covering APIs, configuration files, and removed features. + +7. **Update Type Definitions**: Install latest TypeScript type packages if your project uses TypeScript. + +8. **Validate Changes**: Run build and dev commands to verify functionality works correctly post-upgrade. + +--- + +## v14 → v15 Breaking Changes: What to Look For + +When reviewing code for v14→v15 migration, read each source file and check for +these **specific patterns**. These are the most-missed issues in manual reviews. + +### `params` and `searchParams` are now async (v15 breaking change) + +The most commonly overlooked change. **Read every Page and Layout component** +and check the prop type declaration, not just the usage. + +```tsx +// BAD — v14 style, breaks in v15 with TypeScript errors and runtime warnings +export default async function Page({ + params, +}: { + params: { id: string } // ← WRONG: not a Promise +}) { + const id = params.id // ← WRONG: accessing directly without await + return
{id}
+} + +// GOOD — v15 style +export default async function Page({ + params, +}: { + params: Promise<{ id: string }> // ← Promise<> wrapper required +}) { + const { id } = await params // ← must await before accessing + return
{id}
+} +``` + +Same pattern applies to `searchParams`: + +```tsx +// BAD — v14 style +export default async function Page({ + searchParams, +}: { + searchParams: { q?: string } // ← WRONG +}) { + const query = searchParams.q // ← WRONG + +// GOOD — v15 style +export default async function Page({ + searchParams, +}: { + searchParams: Promise<{ q?: string }> // ← Promise<> wrapper required +}) { + const { q } = await searchParams // ← must await +``` + +### `cookies()` and `headers()` are now async (v15 breaking change) + +```tsx +// BAD +const cookieStore = cookies() // ← WRONG: synchronous +const headersList = headers() // ← WRONG: synchronous + +// GOOD +const cookieStore = await cookies() // ← await required +const headersList = await headers() // ← await required +``` + +### `viewport` must be a separate export (removed from `metadata`) + +```tsx +// BAD +export const metadata: Metadata = { + title: 'My App', + viewport: { width: 'device-width' }, // ← WRONG: viewport inside metadata +} + +// GOOD +export const metadata: Metadata = { title: 'My App' } +export const viewport: Viewport = { width: 'device-width' } // ← separate export +``` + +### `package.json` version + +Update `next` to `^15.0.0` and `react`/`react-dom` to `^19.0.0`. diff --git a/examples/workbench/next-upgrade/references/next-upgrade/upgrade-guide.md b/examples/workbench/next-upgrade/references/next-upgrade/upgrade-guide.md new file mode 100644 index 0000000..5221327 --- /dev/null +++ b/examples/workbench/next-upgrade/references/next-upgrade/upgrade-guide.md @@ -0,0 +1,178 @@ +# Next.js v14 → v15 Upgrade Guide (Vendored Reference) + +This is a vendored snapshot of the key v14→v15 breaking changes and migration steps. + +--- + +## 1. Package Version + +Update `next` in `package.json`: + +```bash +npm install next@15 react@19 react-dom@19 +``` + +If you cannot run npm, manually update the version field: +```json +"next": "^15.0.0" +``` + +--- + +## 2. Async Request APIs (BREAKING) + +In v15, the following APIs are **asynchronous** and must be awaited. Previously they were synchronous. + +### `cookies()` and `headers()` + +```tsx +// v14 (synchronous — now broken in v15) +import { cookies, headers } from 'next/headers' +const cookieStore = cookies() +const headersList = headers() + +// v15 (async — must await) +import { cookies, headers } from 'next/headers' +const cookieStore = await cookies() +const headersList = await headers() +``` + +### `params` and `searchParams` in Page/Layout components + +```tsx +// v14 (synchronous — now broken in v15) +export default function Page({ params }: { params: { id: string } }) { + const id = params.id + return
{id}
+} + +// v15 (async — must await) +export default async function Page({ + params, +}: { + params: Promise<{ id: string }> +}) { + const { id } = await params + return
{id}
+} +``` + +Same pattern applies to `searchParams`: + +```tsx +// v14 +export default function Page({ searchParams }: { searchParams: { q?: string } }) { + const query = searchParams.q +} + +// v15 +export default async function Page({ + searchParams, +}: { + searchParams: Promise<{ q?: string }> +}) { + const { q } = await searchParams +} +``` + +### Automated codemod + +The official codemod handles most async-request-api changes: + +```bash +npx @next/codemod@canary next-async-request-api . +``` + +Or use the upgrade CLI (runs multiple codemods): + +```bash +npx @next/codemod@canary upgrade +``` + +--- + +## 3. `draftMode()` is now async + +```tsx +// v14 +import { draftMode } from 'next/headers' +const { isEnabled } = draftMode() + +// v15 +import { draftMode } from 'next/headers' +const { isEnabled } = await draftMode() +``` + +--- + +## 4. Fetch Caching Changes + +In v15, `fetch()` requests are **no longer cached by default**. + +```tsx +// v14: cached by default (equivalent to cache: 'force-cache') +const res = await fetch('https://api.example.com/data') + +// v15: NOT cached by default (equivalent to cache: 'no-store') +// To opt in to caching: +const res = await fetch('https://api.example.com/data', { cache: 'force-cache' }) +``` + +--- + +## 5. TypeScript: Updated `PageProps` and `LayoutProps` + +With async params/searchParams, TypeScript types change: + +```tsx +// v14 +type Props = { + params: { id: string } + searchParams: { [key: string]: string | string[] | undefined } +} + +// v15 +type Props = { + params: Promise<{ id: string }> + searchParams: Promise<{ [key: string]: string | string[] | undefined }> +} +``` + +--- + +## 6. Viewport Metadata (if not yet updated from v13) + +If you are still using `viewport` inside the `metadata` export, move it to a separate `viewport` export: + +```tsx +// Deprecated in v13.4, removed in v15 +export const metadata: Metadata = { + title: 'My App', + viewport: { width: 'device-width', initialScale: 1 }, +} + +// v15-compatible +import type { Metadata, Viewport } from 'next' + +export const metadata: Metadata = { + title: 'My App', +} + +export const viewport: Viewport = { + width: 'device-width', + initialScale: 1, +} +``` + +--- + +## Summary Checklist + +- [ ] Update `next` package to v15 in `package.json` +- [ ] Await `cookies()` and `headers()` in all Server Components, Route Handlers, Middleware +- [ ] Make `params` a `Promise<{...}>` and `await params` in pages/layouts +- [ ] Make `searchParams` a `Promise<{...}>` and `await searchParams` in pages +- [ ] Await `draftMode()` if used +- [ ] Review any `fetch()` calls that relied on default caching +- [ ] Move `viewport` out of `metadata` export if present +- [ ] Run `npm run build` to surface remaining type errors diff --git a/examples/workbench/next-upgrade/suite.yml b/examples/workbench/next-upgrade/suite.yml new file mode 100644 index 0000000..35fc0d3 --- /dev/null +++ b/examples/workbench/next-upgrade/suite.yml @@ -0,0 +1,40 @@ +name: next-upgrade-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4-5 + - openrouter/openai/gpt-4o-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + Keep all task outputs within /work unless asked otherwise. +cases: + - name: review-starter-app + workspace: ./workspace + task: | + You have a Next.js 14 project at /work/starter-app. + + Use the next-upgrade skill at /work/references/next-upgrade/SKILL.md + to review this project for all v14→v15 migration issues. The upgrade + guide is vendored at /work/references/next-upgrade/upgrade-guide.md. + + Write your findings to /work/findings.txt. For each issue found, + write one line in this exact format: + : + + Examples of valid lines: + app/api/route.ts:5 cookies() must be awaited in v15 + app/page.tsx:6 viewport should be a separate export + package.json:10 next version should be upgraded to v15 + + Review ALL files in the project and report every v14→v15 issue you find. + graders: + - name: findings-page + command: node $CASE/checks/grade-page-findings.mjs + - name: findings-id-page + command: node $CASE/checks/grade-id-page-findings.mjs + - name: findings-route + command: node $CASE/checks/grade-route-findings.mjs + - name: findings-package + command: node $CASE/checks/grade-package-findings.mjs diff --git a/examples/workbench/next-upgrade/workspace/starter-app/app/[id]/page.tsx b/examples/workbench/next-upgrade/workspace/starter-app/app/[id]/page.tsx new file mode 100644 index 0000000..105a5f3 --- /dev/null +++ b/examples/workbench/next-upgrade/workspace/starter-app/app/[id]/page.tsx @@ -0,0 +1,14 @@ +export default async function ItemPage({ + params, +}: { + params: { id: string } +}) { + const id = params.id + + return ( +
+

Item {id}

+

Viewing details for item {id}.

+
+ ) +} diff --git a/examples/workbench/next-upgrade/workspace/starter-app/app/api/route.ts b/examples/workbench/next-upgrade/workspace/starter-app/app/api/route.ts new file mode 100644 index 0000000..4715a9e --- /dev/null +++ b/examples/workbench/next-upgrade/workspace/starter-app/app/api/route.ts @@ -0,0 +1,17 @@ +import { cookies, headers } from 'next/headers' +import { NextResponse } from 'next/server' + +export async function GET() { + const cookieStore = cookies() + const headersList = headers() + + const token = cookieStore.get('auth-token') + const host = headersList.get('host') + const userAgent = headersList.get('user-agent') + + return NextResponse.json({ + authenticated: !!token, + host, + userAgent, + }) +} diff --git a/examples/workbench/next-upgrade/workspace/starter-app/app/page.tsx b/examples/workbench/next-upgrade/workspace/starter-app/app/page.tsx new file mode 100644 index 0000000..0e8382d --- /dev/null +++ b/examples/workbench/next-upgrade/workspace/starter-app/app/page.tsx @@ -0,0 +1,27 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'My App', + description: 'A sample Next.js application', + viewport: { + width: 'device-width', + initialScale: 1, + }, +} + +export default async function Home({ + searchParams, +}: { + searchParams: { query?: string; page?: string } +}) { + const query = searchParams.query ?? '' + const page = searchParams.page ?? '1' + + return ( +
+

Search Results

+

Query: {query}

+

Page: {page}

+
+ ) +} diff --git a/examples/workbench/next-upgrade/workspace/starter-app/package.json b/examples/workbench/next-upgrade/workspace/starter-app/package.json new file mode 100644 index 0000000..c48da0b --- /dev/null +++ b/examples/workbench/next-upgrade/workspace/starter-app/package.json @@ -0,0 +1,22 @@ +{ + "name": "my-next-app", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint" + }, + "dependencies": { + "next": "14.2.5", + "react": "^18.3.0", + "react-dom": "^18.3.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^18", + "@types/react-dom": "^18", + "typescript": "^5" + } +} diff --git a/examples/workbench/pptx/README.md b/examples/workbench/pptx/README.md new file mode 100644 index 0000000..106cbf8 --- /dev/null +++ b/examples/workbench/pptx/README.md @@ -0,0 +1,58 @@ +# pptx skill eval + +Eval suite for +[`anthropics/skills/pptx`](https://github.com/anthropics/skills) — +skill for reading, editing, and creating PowerPoint presentations using +markitdown, pptxgenjs, and an unpack/edit/pack XML workflow. + +## Cases + +### `extract-pptx-facts` — read and extract structured data + +Sample: `presentation.pptx` (4-slide TechVision Corp Q3 2025 deck, created by setup) + +| Field | Expected value | Maps to | +|---|---|---| +| `title` | `"TechVision Corp: Q3 2025 Results"` | title text on slide 1 | +| `slideCount` | `4` | total slides | +| `revenue` | `"$5.1M"` | financial highlights slide | +| `customerCount` | `2341` | customer metrics slide | + +### `create-product-deck` — create from scratch with pptxgenjs + +Task: build `deck.pptx` for NovaSoft Analytics with 4 slides. + +| Required string | Slide | Rule | +|---|---|---| +| `NovaSoft Analytics` | 1 — title | company name present | +| `Smarter Business Decisions` | 1 — subtitle | exact title subtitle | +| `Key Features` | 2 — heading | features slide heading | +| `40%` | 3 — stat callout | proven-results statistic | +| `novasoft.io` | 4 — CTA | closing call-to-action URL | + +### `no-pptx-skill-needed` — control case + +Writes `answer.txt` with a literal string. Grader fails if the agent +reads `pptx/SKILL.md` unnecessarily. + +## Vendored snapshot + +The skill normally references `editing.md` and `pptxgenjs.md` as +relative file links within the same directory. For deterministic eval +these docs are vendored at `references/pptx/` alongside `SKILL.md`. +No WebFetch calls are needed — the diff vs upstream is zero lines. + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-6` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/pptx/analysis.md b/examples/workbench/pptx/analysis.md new file mode 100644 index 0000000..cd52ef1 --- /dev/null +++ b/examples/workbench/pptx/analysis.md @@ -0,0 +1,18 @@ +--- +skill: anthropics/skills/pptx +status: success +classification: document-producer +baseline_rule_coverage: 0.85 +final_rule_coverage: 0.85 +modifications_tried: 0 +total_cost_usd: 0.64 +--- + +# Auto-pilot run for `anthropics/skills/pptx` + +- Classified as **document-producer**: the skill guides agents to produce PPTX files via `pptxgenjs` (scratch) or unpack/edit/pack XML workflow (templates), and read them via `python -m markitdown`. +- Three eval cases: `extract-pptx-facts` (read a 4-slide deck → answer.json), `create-product-deck` (create deck.pptx from scratch with 5 required content strings), `no-pptx-skill-needed` (control case, skill must not be read unnecessarily). +- Setup uses `bash -c "source /work/.venv/bin/activate && pip install --no-cache-dir ..."` to work around `PIP_REQUIRE_VIRTUALENV=1` and `XDG_CACHE_HOME=/work/.cache` in the workbench Docker image. +- **Grader iteration 0 (not counted):** discovered that pptxgenjs renders styled headings as separate `` runs (e.g. "Key" bold + "Features" normal). Fixed grader to trim and join runs with a space instead of newlines. This lifted `create-product-deck` from 5/9 to 8/9 pass. +- **Baseline after calibration:** 23/27 = 0.85 overall. Remaining failure: `gpt-4o-mini` on `extract-pptx-facts` (0/3) — the model never reads `pptx/SKILL.md`, never uses `markitdown`, and cannot parse the binary PPTX. Claude and Gemini pass 3/3 on both creation and extraction cases. +- No upstream skill modifications proposed: baseline already ≥ 0.85 and the gpt-4o-mini failure is a model-capability gap (does not follow appendSystemPrompt guidance), not a gap in the skill text itself. diff --git a/examples/workbench/pptx/checks/_grader-utils.mjs b/examples/workbench/pptx/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/pptx/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/pptx/checks/_trace.mjs b/examples/workbench/pptx/checks/_trace.mjs new file mode 100644 index 0000000..60aea18 --- /dev/null +++ b/examples/workbench/pptx/checks/_trace.mjs @@ -0,0 +1,60 @@ +import { readFileSync } from 'node:fs'; + +export function readTraceJsonl(tracePath) { + return readFileSync(tracePath, 'utf-8') + .trim() + .split(/\r?\n/) + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); +} + +function readPathFromToolCall(entry) { + if (entry?.type !== 'tool_call' || entry.name !== 'read') { + return undefined; + } + const args = entry.arguments; + if (!args || typeof args !== 'object') { + return undefined; + } + if (typeof args.path === 'string') return args.path; + if (typeof args.filePath === 'string') return args.filePath; + return undefined; +} + +function matchesPath(path, pattern) { + if (pattern instanceof RegExp) { + return pattern.test(path); + } + return path === String(pattern); +} + +export function noReadPath(tracePath, forbiddenPath) { + const forbidden = readTraceJsonl(tracePath) + .map(readPathFromToolCall) + .filter((path) => typeof path === 'string' && matchesPath(path, forbiddenPath)); + + if (forbidden.length > 0) { + return { + pass: false, + score: 0, + evidence: forbidden.map((path) => `forbidden read path: ${path}`), + }; + } + + return { + pass: true, + score: 1, + evidence: ['no forbidden read paths found'], + }; +} + +export function printResult(result) { + console.log(JSON.stringify(result)); + process.exit(result.pass ? 0 : 1); +} diff --git a/examples/workbench/pptx/checks/create-inputs.py b/examples/workbench/pptx/checks/create-inputs.py new file mode 100644 index 0000000..36ac920 --- /dev/null +++ b/examples/workbench/pptx/checks/create-inputs.py @@ -0,0 +1,52 @@ +""" +Creates presentation.pptx — a 4-slide TechVision Corp Q3 2025 deck. + +Used as input for the extract-pptx-facts case. The expected answer.json is: + { + "title": "TechVision Corp: Q3 2025 Results", + "slideCount": 4, + "revenue": "$5.1M", + "customerCount": 2341 + } +""" + +import os +from pptx import Presentation +from pptx.util import Inches + +prs = Presentation() +blank = prs.slide_layouts[6] # Blank layout + + +def add_slide(prs, layout, title_text, body_lines=None): + slide = prs.slides.add_slide(layout) + title_box = slide.shapes.add_textbox(Inches(0.5), Inches(0.3), Inches(9), Inches(0.9)) + title_box.text_frame.text = title_text + if body_lines: + body_box = slide.shapes.add_textbox(Inches(0.5), Inches(1.5), Inches(9), Inches(3.5)) + tf = body_box.text_frame + tf.text = body_lines[0] + for line in body_lines[1:]: + tf.add_paragraph().text = line + return slide + + +add_slide(prs, blank, + 'TechVision Corp: Q3 2025 Results', + ['Quarterly Business Review', 'October 15, 2025']) + +add_slide(prs, blank, + 'Financial Highlights', + ['Revenue: $5.1M', 'Growth: 18% YoY', 'Operating Margin: 24%']) + +add_slide(prs, blank, + 'Customer Metrics', + ['Total Customers: 2,341', 'New Customers: 312', 'Churn Rate: 1.8%']) + +add_slide(prs, blank, + 'Looking Ahead', + ['Q4 Target: $6.2M', 'New Product Launch: November', 'Geographic Expansion: EMEA']) + +output_path = os.path.join(os.environ.get('WORK', '/work'), 'presentation.pptx') +prs.save(output_path) +print(f'Created {output_path} with {len(prs.slides)} slides') diff --git a/examples/workbench/pptx/checks/create-product-deck.mjs b/examples/workbench/pptx/checks/create-product-deck.mjs new file mode 100644 index 0000000..a782680 --- /dev/null +++ b/examples/workbench/pptx/checks/create-product-deck.mjs @@ -0,0 +1,90 @@ +/** + * Grader for create-product-deck case. + * + * Checks that deck.pptx: + * 1. Exists and is a valid ZIP/PPTX + * 2. Contains all 5 required content strings in slide XML text nodes + * + * Required strings (must be present verbatim): + * - "NovaSoft Analytics" (company name, title slide) + * - "Smarter Business Decisions" (title slide subtitle) + * - "Key Features" (features slide heading) + * - "40%" (proven results statistic) + * - "novasoft.io" (call-to-action URL, closing slide) + */ + +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { execSync } from 'node:child_process'; +import { join } from 'node:path'; + +const WORK = process.env.WORK ?? '/work'; +const filePath = join(WORK, 'deck.pptx'); + +function emitResult(pass, score, evidence) { + console.log(JSON.stringify({ pass, score, evidence })); + process.exit(pass ? 0 : 1); +} + +if (!existsSync(filePath)) { + emitResult(false, 0, ['deck.pptx was not created']); +} + +// Validate it is a ZIP (all PPTX files are ZIP archives) +try { + execSync(`unzip -t "${filePath}"`, { stdio: 'pipe' }); +} catch { + emitResult(false, 0, ['deck.pptx is not a valid ZIP/PPTX file']); +} + +// Extract slides to a temp directory +const tmpDir = `/tmp/pptx-check-deck-${Date.now()}`; +try { + mkdirSync(tmpDir, { recursive: true }); + execSync(`unzip -q "${filePath}" -d "${tmpDir}"`, { stdio: 'pipe' }); +} catch { + // unzip exits 1 on warnings but may still extract — continue +} + +// Collect all text from slide XML files +let allXml = ''; +try { + allXml = execSync( + `find "${tmpDir}/ppt/slides" -name "*.xml" -not -path "*/_rels/*" -exec cat {} \\;`, + { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'] }, + ); +} catch { + allXml = ''; +} + +// Cleanup temp dir +try { rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ } + +// Extract text content from elements. +// pptxgenjs sometimes splits styled headings across multiple runs in the same +// paragraph (e.g. "Key" bold + " Features" normal = two nodes). Trim +// each run and join with a single space so adjacent runs reconstruct the full +// visible text. — G3: per-finding-line keyword matching lesson. +const textLines = allXml + .split(/]*>/) + .slice(1) + .map((chunk) => chunk.split('')[0].trim()) + .filter(Boolean); +const extractedText = textLines.join(' '); + +const required = [ + { id: 'company-name', term: 'NovaSoft Analytics', desc: 'company name (title slide)' }, + { id: 'title-subtitle', term: 'Smarter Business Decisions', desc: 'subtitle (title slide)' }, + { id: 'features-heading', term: 'Key Features', desc: 'features slide heading' }, + { id: 'stat-40pct', term: '40%', desc: 'proven-results statistic' }, + { id: 'cta-url', term: 'novasoft.io', desc: 'call-to-action URL (closing slide)' }, +]; + +const found = required.filter((r) => extractedText.includes(r.term)); +const missing = required.filter((r) => !extractedText.includes(r.term)); +const pass = found.length === required.length; + +emitResult(pass, found.length / required.length, [ + `${found.length}/${required.length} required content items found in deck.pptx`, + ...found.map((r) => `+ ${r.id}: "${r.term}"`), + ...missing.map((r) => `- ${r.id}: missing "${r.term}" (${r.desc})`), +]); diff --git a/examples/workbench/pptx/checks/extract-pptx-facts.mjs b/examples/workbench/pptx/checks/extract-pptx-facts.mjs new file mode 100644 index 0000000..a24103a --- /dev/null +++ b/examples/workbench/pptx/checks/extract-pptx-facts.mjs @@ -0,0 +1,74 @@ +/** + * Grader for extract-pptx-facts case. + * + * Expected answer.json: + * { "title": "TechVision Corp: Q3 2025 Results", "slideCount": 4, + * "revenue": "$5.1M", "customerCount": 2341 } + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const WORK = process.env.WORK ?? '/work'; +const answerPath = join(WORK, 'answer.json'); + +function emitResult(pass, evidence) { + const score = pass ? 1 : 0; + console.log(JSON.stringify({ pass, score, evidence: Array.isArray(evidence) ? evidence : [String(evidence)] })); + process.exit(pass ? 0 : 1); +} + +if (!existsSync(answerPath)) { + emitResult(false, ['answer.json was not created']); +} + +let answer; +try { + answer = JSON.parse(readFileSync(answerPath, 'utf-8')); +} catch (err) { + emitResult(false, [`answer.json is not valid JSON: ${err.message}`]); +} + +const failures = []; + +// title — must mention TechVision Corp and Q3 2025 +if (typeof answer.title !== 'string') { + failures.push('title must be a string'); +} else if (!answer.title.includes('TechVision Corp')) { + failures.push(`title must include "TechVision Corp", got: "${answer.title}"`); +} else if (!answer.title.includes('Q3 2025')) { + failures.push(`title must include "Q3 2025", got: "${answer.title}"`); +} + +// slideCount — must be exactly 4 +if (answer.slideCount !== 4) { + failures.push(`slideCount must be 4, got: ${JSON.stringify(answer.slideCount)}`); +} + +// revenue — must contain "5.1" (accept "$5.1M", "5.1M", "$5.1 million", etc.) +const revenueStr = String(answer.revenue ?? ''); +if (!revenueStr.includes('5.1')) { + failures.push(`revenue must include "5.1", got: "${revenueStr}"`); +} + +// customerCount — must resolve to 2341 (accept number 2341 or string "2341" or "2,341") +const rawCount = answer.customerCount; +const countNum = typeof rawCount === 'number' + ? rawCount + : Number(String(rawCount ?? '').replace(/,/g, '')); +if (countNum !== 2341) { + failures.push(`customerCount must resolve to 2341, got: ${JSON.stringify(rawCount)}`); +} + +const pass = failures.length === 0; +const evidence = pass + ? [ + 'answer.json matched all expected fields', + `+ title: "${answer.title}"`, + `+ slideCount: ${answer.slideCount}`, + `+ revenue: "${answer.revenue}"`, + `+ customerCount: ${answer.customerCount}`, + ] + : failures; + +emitResult(pass, evidence); diff --git a/examples/workbench/pptx/checks/no-pptx-skill.mjs b/examples/workbench/pptx/checks/no-pptx-skill.mjs new file mode 100644 index 0000000..7dbb6d4 --- /dev/null +++ b/examples/workbench/pptx/checks/no-pptx-skill.mjs @@ -0,0 +1,41 @@ +/** + * Grader for no-pptx-skill-needed case. + * + * Checks: + * 1. answer.txt was created with exactly "Q4 Revenue: $8.7M" + * 2. The agent did NOT read the pptx SKILL.md (no skill needed for this task) + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { noReadPath, printResult } from './_trace.mjs'; + +const WORK = process.env.WORK ?? '/work'; +const RESULTS = process.env.RESULTS ?? '/results'; + +const answerPath = join(WORK, 'answer.txt'); +const tracePath = join(RESULTS, 'trace.jsonl'); +const failures = []; + +if (!existsSync(answerPath)) { + failures.push('answer.txt was not created'); +} else { + const content = readFileSync(answerPath, 'utf-8').trim(); + if (content !== 'Q4 Revenue: $8.7M') { + failures.push(`answer.txt must contain exactly "Q4 Revenue: $8.7M", got: "${content}"`); + } +} + +if (existsSync(tracePath)) { + const traceResult = noReadPath(tracePath, /\/pptx\/SKILL\.md$/); + if (!traceResult.pass) { + failures.push(...traceResult.evidence); + } +} + +printResult( + failures.length === 0 + ? { pass: true, score: 1, evidence: ['answer.txt correct and pptx skill was not read'] } + : { pass: false, score: 0, evidence: failures }, +); diff --git a/examples/workbench/pptx/references/pptx/SKILL.md b/examples/workbench/pptx/references/pptx/SKILL.md new file mode 100644 index 0000000..19c7085 --- /dev/null +++ b/examples/workbench/pptx/references/pptx/SKILL.md @@ -0,0 +1,157 @@ +# PPTX Skill + +## Quick Reference + +| Task | Guide | +|------|-------| +| Read/analyze content | `python -m markitdown presentation.pptx` | +| Edit or create from template | Read [editing.md](editing.md) | +| Create from scratch | Read [pptxgenjs.md](pptxgenjs.md) | + +--- + +## Reading Content + +```bash +# Text extraction +python -m markitdown presentation.pptx + +# Visual overview +python scripts/thumbnail.py presentation.pptx + +# Raw XML +python scripts/office/unpack.py presentation.pptx unpacked/ +``` + +--- + +## Editing Workflow + +**Read [editing.md](editing.md) for full details.** + +1. Analyze template with `thumbnail.py` +2. Unpack → manipulate slides → edit content → clean → pack + +--- + +## Creating from Scratch + +**Read [pptxgenjs.md](pptxgenjs.md) for full details.** + +Use when no template or reference presentation is available. + +--- + +## Design Ideas + +**Don't create boring slides.** "Plain bullets on a white background won't impress anyone." Consider these principles: + +### Before Starting + +- **Pick a bold, content-informed color palette**: "The palette should feel designed for THIS topic." +- **Dominance over equality**: "One color should dominate (60-70% visual weight), with 1-2 supporting tones and one sharp accent." +- **Dark/light contrast**: Dark backgrounds for titles/conclusions, light for content (sandwich structure) +- **Commit to a visual motif**: "Pick ONE distinctive element and repeat it across every slide" + +### Color Palettes + +| Theme | Primary | Secondary | Accent | +|-------|---------|-----------|--------| +| **Midnight Executive** | `1E2761` (navy) | `CADCFC` (ice blue) | `FFFFFF` (white) | +| **Forest & Moss** | `2C5F2D` (forest) | `97BC62` (moss) | `F5F5F5` (cream) | +| **Coral Energy** | `F96167` (coral) | `F9E795` (gold) | `2F3C7E` (navy) | +| **Warm Terracotta** | `B85042` (terracotta) | `E7E8D1` (sand) | `A7BEAE` (sage) | +| **Ocean Gradient** | `065A82` (deep blue) | `1C7293` (teal) | `21295C` (midnight) | +| **Charcoal Minimal** | `36454F` (charcoal) | `F2F2F2` (off-white) | `212121` (black) | +| **Teal Trust** | `028090` (teal) | `00A896` (seafoam) | `02C39A` (mint) | +| **Berry & Cream** | `6D2E46` (berry) | `A26769` (dusty rose) | `ECE2D0` (cream) | +| **Sage Calm** | `84B59F` (sage) | `69A297` (eucalyptus) | `50808E` (slate) | +| **Cherry Bold** | `990011` (cherry) | `FCF6F5` (off-white) | `2F3C7E` (navy) | + +### For Each Slide + +"Every slide needs a visual element — image, chart, icon, or shape. Text-only slides are forgettable." + +Layout options include two-column, icon + text rows, grids, and half-bleed images. Use large stat callouts, comparison columns, and process flows for data. + +### Typography + +| Header Font | Body Font | +|-------------|-----------| +| Georgia | Calibri | +| Arial Black | Arial | +| Calibri | Calibri Light | +| Cambria | Calibri | +| Trebuchet MS | Calibri | +| Impact | Arial | +| Palatino | Garamond | +| Consolas | Calibri | + +| Element | Size | +|---------|------| +| Slide title | 36-44pt bold | +| Section header | 20-24pt bold | +| Body text | 14-16pt | +| Captions | 10-12pt muted | + +### Spacing + +- 0.5" minimum margins +- 0.3-0.5" between content blocks +- Leave breathing room + +### Avoid (Common Mistakes) + +"Don't repeat the same layout," center body text, skimp on size contrast, default to blue, or create text-only slides. "NEVER use accent lines under titles — these are a hallmark of AI-generated slides." + +--- + +## QA (Required) + +"Assume there are problems. Your job is to find them. Your first render is almost never correct." + +### Content QA + +```bash +python -m markitdown output.pptx +python -m markitdown output.pptx | grep -iE "xxxx|lorem|ipsum|this.*(page|slide).*layout" +``` + +### Visual QA + +"⚠️ USE SUBAGENTS — even for 2-3 slides. You've been staring at the code and will see what you expect, not what's there." + +Convert slides to images and inspect for overlapping elements, text overflow, alignment issues, contrast problems, and placeholder content. + +### Verification Loop + +1. Generate → Convert to images → Inspect +2. List issues found +3. Fix issues +4. Re-verify affected slides +5. Repeat until clean pass + +"Do not declare success until you've completed at least one fix-and-verify cycle." + +--- + +## Converting to Images + +```bash +python scripts/office/soffice.py --headless --convert-to pdf output.pptx +pdftoppm -jpeg -r 150 output.pdf slide +``` + +--- + +## Dependencies + +- `pip install "markitdown[pptx]"` - text extraction +- `pip install Pillow` - thumbnails +- `npm install -g pptxgenjs` - creating from scratch +- LibreOffice (`soffice`) - PDF conversion +- Poppler (`pdftoppm`) - PDF to images + +--- + +**License:** Proprietary. LICENSE.txt contains complete terms. diff --git a/examples/workbench/pptx/references/pptx/editing.md b/examples/workbench/pptx/references/pptx/editing.md new file mode 100644 index 0000000..27bf190 --- /dev/null +++ b/examples/workbench/pptx/references/pptx/editing.md @@ -0,0 +1,205 @@ +# Editing Presentations + +## Template-Based Workflow + +When using an existing presentation as a template: + +1. **Analyze existing slides**: + ```bash + python scripts/thumbnail.py template.pptx + python -m markitdown template.pptx + ``` + Review `thumbnails.jpg` to see layouts, and markitdown output to see placeholder text. + +2. **Plan slide mapping**: For each content section, choose a template slide. + + ⚠️ **USE VARIED LAYOUTS** — monotonous presentations are a common failure mode. Don't default to basic title + bullet slides. Actively seek out: + - Multi-column layouts (2-column, 3-column) + - Image + text combinations + - Full-bleed images with text overlay + - Quote or callout slides + - Section dividers + - Stat/number callouts + - Icon grids or icon + text rows + + **Avoid:** Repeating the same text-heavy layout for every slide. + + Match content type to layout style (e.g., key points → bullet slide, team info → multi-column, testimonials → quote slide). + +3. **Unpack**: `python scripts/office/unpack.py template.pptx unpacked/` + +4. **Build presentation** (do this yourself, not with subagents): + - Delete unwanted slides (remove from ``) + - Duplicate slides you want to reuse (`add_slide.py`) + - Reorder slides in `` + - **Complete all structural changes before step 5** + +5. **Edit content**: Update text in each `slide{N}.xml`. + **Use subagents here if available** — slides are separate XML files, so subagents can edit in parallel. + +6. **Clean**: `python scripts/clean.py unpacked/` + +7. **Pack**: `python scripts/office/pack.py unpacked/ output.pptx --original template.pptx` + +--- + +## Scripts + +| Script | Purpose | +|--------|---------| +| `unpack.py` | Extract and pretty-print PPTX | +| `add_slide.py` | Duplicate slide or create from layout | +| `clean.py` | Remove orphaned files | +| `pack.py` | Repack with validation | +| `thumbnail.py` | Create visual grid of slides | + +### unpack.py + +```bash +python scripts/office/unpack.py input.pptx unpacked/ +``` + +Extracts PPTX, pretty-prints XML, escapes smart quotes. + +### add_slide.py + +```bash +python scripts/add_slide.py unpacked/ slide2.xml # Duplicate slide +python scripts/add_slide.py unpacked/ slideLayout2.xml # From layout +``` + +Prints `` to add to `` at desired position. + +### clean.py + +```bash +python scripts/clean.py unpacked/ +``` + +Removes slides not in ``, unreferenced media, orphaned rels. + +### pack.py + +```bash +python scripts/office/pack.py unpacked/ output.pptx --original input.pptx +``` + +Validates, repairs, condenses XML, re-encodes smart quotes. + +### thumbnail.py + +```bash +python scripts/thumbnail.py input.pptx [output_prefix] [--cols N] +``` + +Creates `thumbnails.jpg` with slide filenames as labels. Default 3 columns, max 12 per grid. + +**Use for template analysis only** (choosing layouts). For visual QA, use `soffice` + `pdftoppm` to create full-resolution individual slide images—see SKILL.md. + +--- + +## Slide Operations + +Slide order is in `ppt/presentation.xml` → ``. + +**Reorder**: Rearrange `` elements. + +**Delete**: Remove ``, then run `clean.py`. + +**Add**: Use `add_slide.py`. Never manually copy slide files—the script handles notes references, Content_Types.xml, and relationship IDs that manual copying misses. + +--- + +## Editing Content + +**Subagents:** If available, use them here (after completing step 4). Each slide is a separate XML file, so subagents can edit in parallel. In your prompt to subagents, include: +- The slide file path(s) to edit +- **"Use the Edit tool for all changes"** +- The formatting rules and common pitfalls below + +For each slide: +1. Read the slide's XML +2. Identify ALL placeholder content—text, images, charts, icons, captions +3. Replace each placeholder with final content + +**Use the Edit tool, not sed or Python scripts.** The Edit tool forces specificity about what to replace and where, yielding better reliability. + +### Formatting Rules + +- **Bold all headers, subheadings, and inline labels**: Use `b="1"` on ``. This includes: + - Slide titles + - Section headers within a slide + - Inline labels like (e.g.: "Status:", "Description:") at the start of a line +- **Never use unicode bullets (•)**: Use proper list formatting with `` or `` +- **Bullet consistency**: Let bullets inherit from the layout. Only specify `` or ``. + +--- + +## Common Pitfalls + +### Template Adaptation + +When source content has fewer items than the template: +- **Remove excess elements entirely** (images, shapes, text boxes), don't just clear text +- Check for orphaned visuals after clearing text content +- Run visual QA to catch mismatched counts + +When replacing text with different length content: +- **Shorter replacements**: Usually safe +- **Longer replacements**: May overflow or wrap unexpectedly +- Test with visual QA after text changes +- Consider truncating or splitting content to fit the template's design constraints + +**Template slots ≠ Source items**: If template has 4 team members but source has 3 users, delete the 4th member's entire group (image + text boxes), not just the text. + +### Multi-Item Content + +If source has multiple items (numbered lists, multiple sections), create separate `` elements for each — **never concatenate into one string**. + +**❌ WRONG** — all items in one paragraph: +```xml + + Step 1: Do the first thing. Step 2: Do the second thing. + +``` + +**✅ CORRECT** — separate paragraphs with bold headers: +```xml + + + Step 1 + + + + Do the first thing. + + + + Step 2 + + +``` + +Copy `` from the original paragraph to preserve line spacing. Use `b="1"` on headers. + +### Smart Quotes + +Handled automatically by unpack/pack. But the Edit tool converts smart quotes to ASCII. + +**When adding new text with quotes, use XML entities:** + +```xml +the “Agreement” +``` + +| Character | Name | Unicode | XML Entity | +|-----------|------|---------|------------| +| `"` | Left double quote | U+201C | `“` | +| `"` | Right double quote | U+201D | `”` | +| `'` | Left single quote | U+2018 | `‘` | +| `'` | Right single quote | U+2019 | `’` | + +### Other + +- **Whitespace**: Use `xml:space="preserve"` on `` with leading/trailing spaces +- **XML parsing**: Use `defusedxml.minidom`, not `xml.etree.ElementTree` (corrupts namespaces) diff --git a/examples/workbench/pptx/references/pptx/pptxgenjs.md b/examples/workbench/pptx/references/pptx/pptxgenjs.md new file mode 100644 index 0000000..37c37bc --- /dev/null +++ b/examples/workbench/pptx/references/pptx/pptxgenjs.md @@ -0,0 +1,422 @@ +# PptxGenJS Tutorial + +## Setup & Basic Structure + +```javascript +const pptxgen = require("pptxgenjs"); + +let pres = new pptxgen(); +pres.layout = 'LAYOUT_16x9'; // or 'LAYOUT_16x10', 'LAYOUT_4x3', 'LAYOUT_WIDE' +pres.author = 'Your Name'; +pres.title = 'Presentation Title'; + +let slide = pres.addSlide(); +slide.addText("Hello World!", { x: 0.5, y: 0.5, fontSize: 36, color: "363636" }); + +pres.writeFile({ fileName: "Presentation.pptx" }); +``` + +## Layout Dimensions + +Slide dimensions (coordinates in inches): +- `LAYOUT_16x9`: 10" × 5.625" (default) +- `LAYOUT_16x10`: 10" × 6.25" +- `LAYOUT_4x3`: 10" × 7.5" +- `LAYOUT_WIDE`: 13.3" × 7.5" + +--- + +## Text & Formatting + +```javascript +// Basic text +slide.addText("Simple Text", { + x: 1, y: 1, w: 8, h: 2, fontSize: 24, fontFace: "Arial", + color: "363636", bold: true, align: "center", valign: "middle" +}); + +// Character spacing (use charSpacing, not letterSpacing which is silently ignored) +slide.addText("SPACED TEXT", { x: 1, y: 1, w: 8, h: 1, charSpacing: 6 }); + +// Rich text arrays +slide.addText([ + { text: "Bold ", options: { bold: true } }, + { text: "Italic ", options: { italic: true } } +], { x: 1, y: 3, w: 8, h: 1 }); + +// Multi-line text (requires breakLine: true) +slide.addText([ + { text: "Line 1", options: { breakLine: true } }, + { text: "Line 2", options: { breakLine: true } }, + { text: "Line 3" } // Last item doesn't need breakLine +], { x: 0.5, y: 0.5, w: 8, h: 2 }); + +// Text box margin (internal padding) +slide.addText("Title", { + x: 0.5, y: 0.3, w: 9, h: 0.6, + margin: 0 // Use 0 when aligning text with other elements like shapes or icons +}); +``` + +**Tip:** "Text boxes have internal margin by default. Set `margin: 0` when you need text to align precisely with shapes, +lines, or icons at the same x-position." + +--- + +## Lists & Bullets + +```javascript +// ✅ CORRECT: Multiple bullets +slide.addText([ + { text: "First item", options: { bullet: true, breakLine: true } }, + { text: "Second item", options: { bullet: true, breakLine: true } }, + { text: "Third item", options: { bullet: true } } +], { x: 0.5, y: 0.5, w: 8, h: 3 }); + +// ❌ WRONG: Never use unicode bullets +slide.addText("• First item", { ... }); // Creates double bullets + +// Sub-items and numbered lists +{ text: "Sub-item", options: { bullet: true, indentLevel: 1 } } +{ text: "First", options: { bullet: { type: "number" }, breakLine: true } } +``` + +--- + +## Shapes + +```javascript +slide.addShape(pres.shapes.RECTANGLE, { + x: 0.5, y: 0.8, w: 1.5, h: 3.0, + fill: { color: "FF0000" }, line: { color: "000000", width: 2 } +}); + +slide.addShape(pres.shapes.OVAL, { x: 4, y: 1, w: 2, h: 2, fill: { color: "0000FF" } }); + +slide.addShape(pres.shapes.LINE, { + x: 1, y: 3, w: 5, h: 0, line: { color: "FF0000", width: 3, dashType: "dash" } +}); + +// With transparency +slide.addShape(pres.shapes.RECTANGLE, { + x: 1, y: 1, w: 3, h: 2, + fill: { color: "0088CC", transparency: 50 } +}); + +// Rounded rectangle (rectRadius only works with ROUNDED_RECTANGLE, not RECTANGLE) +// ⚠️ Don't pair with rectangular accent overlays — they won't cover rounded corners. Use RECTANGLE instead. +slide.addShape(pres.shapes.ROUNDED_RECTANGLE, { + x: 1, y: 1, w: 3, h: 2, + fill: { color: "FFFFFF" }, rectRadius: 0.1 +}); + +// With shadow +slide.addShape(pres.shapes.RECTANGLE, { + x: 1, y: 1, w: 3, h: 2, + fill: { color: "FFFFFF" }, + shadow: { type: "outer", color: "000000", blur: 6, offset: 2, angle: 135, opacity: 0.15 } +}); +``` + +Shadow options: + +| Property | Type | Range | Notes | +|----------|------|-------|-------| +| `type` | string | `"outer"`, `"inner"` | | +| `color` | string | 6-char hex (e.g. `"000000"`) | No `#` prefix, no 8-char hex — see Common Pitfalls | +| `blur` | number | 0-100 pt | | +| `offset` | number | 0-200 pt | **Must be non-negative** — negative values corrupt the file | +| `angle` | number | 0-359 degrees | Direction the shadow falls (135 = bottom-right, 270 = upward) | +| `opacity` | number | 0.0-1.0 | Use this for transparency, never encode in color string | + +To cast a shadow upward (e.g. on a footer bar), use `angle: 270` with a positive offset — do **not** use a negative offset. + +**Note**: Gradient fills are not natively supported. Use a gradient image as a background instead. + +--- + +## Images + +### Image Sources + +```javascript +// From file path +slide.addImage({ path: "images/chart.png", x: 1, y: 1, w: 5, h: 3 }); + +// From URL +slide.addImage({ path: "https://example.com/image.jpg", x: 1, y: 1, w: 5, h: 3 }); + +// From base64 (faster, no file I/O) +slide.addImage({ data: "image/png;base64,iVBORw0KGgo...", x: 1, y: 1, w: 5, h: 3 }); +``` + +### Image Options + +```javascript +slide.addImage({ + path: "image.png", + x: 1, y: 1, w: 5, h: 3, + rotate: 45, // 0-359 degrees + rounding: true, // Circular crop + transparency: 50, // 0-100 + flipH: true, // Horizontal flip + flipV: false, // Vertical flip + altText: "Description", // Accessibility + hyperlink: { url: "https://example.com" } +}); +``` + +### Image Sizing Modes + +```javascript +// Contain - fit inside, preserve ratio +{ sizing: { type: 'contain', w: 4, h: 3 } } + +// Cover - fill area, preserve ratio (may crop) +{ sizing: { type: 'cover', w: 4, h: 3 } } + +// Crop - cut specific portion +{ sizing: { type: 'crop', x: 0.5, y: 0.5, w: 2, h: 2 } } +``` + +### Calculate Dimensions (preserve aspect ratio) + +```javascript +const origWidth = 1978, origHeight = 923, maxHeight = 3.0; +const calcWidth = maxHeight * (origWidth / origHeight); +const centerX = (10 - calcWidth) / 2; + +slide.addImage({ path: "image.png", x: centerX, y: 1.2, w: calcWidth, h: maxHeight }); +``` + +### Supported Formats + +- **Standard**: PNG, JPG, GIF (animated GIFs work in Microsoft 365) +- **SVG**: Works in modern PowerPoint/Microsoft 365 + +--- + +## Icons + +Use react-icons to generate SVG icons, then rasterize to PNG for universal compatibility. + +### Setup + +```javascript +const React = require("react"); +const ReactDOMServer = require("react-dom/server"); +const sharp = require("sharp"); +const { FaCheckCircle, FaChartLine } = require("react-icons/fa"); + +function renderIconSvg(IconComponent, color = "#000000", size = 256) { + return ReactDOMServer.renderToStaticMarkup( + React.createElement(IconComponent, { color, size: String(size) }) + ); +} + +async function iconToBase64Png(IconComponent, color, size = 256) { + const svg = renderIconSvg(IconComponent, color, size); + const pngBuffer = await sharp(Buffer.from(svg)).png().toBuffer(); + return "image/png;base64," + pngBuffer.toString("base64"); +} +``` + +### Add Icon to Slide + +```javascript +const iconData = await iconToBase64Png(FaCheckCircle, "#4472C4", 256); + +slide.addImage({ + data: iconData, + x: 1, y: 1, w: 0.5, h: 0.5 // Size in inches +}); +``` + +**Note**: "Use size 256 or higher for crisp icons. The size parameter controls the rasterization resolution, not the +display size on the slide (which is set by `w` and `h` in inches)." + +### Icon Libraries + +Install: `npm install -g react-icons react react-dom sharp` + +Popular icon sets in react-icons: +- `react-icons/fa` - Font Awesome +- `react-icons/md` - Material Design +- `react-icons/hi` - Heroicons +- `react-icons/bi` - Bootstrap Icons + +--- + +## Slide Backgrounds + +```javascript +// Solid color +slide.background = { color: "F1F1F1" }; + +// Color with transparency +slide.background = { color: "FF3399", transparency: 50 }; + +// Image from URL +slide.background = { path: "https://example.com/bg.jpg" }; + +// Image from base64 +slide.background = { data: "image/png;base64,iVBORw0KGgo..." }; +``` + +--- + +## Tables + +```javascript +slide.addTable([ + ["Header 1", "Header 2"], + ["Cell 1", "Cell 2"] +], { + x: 1, y: 1, w: 8, h: 2, + border: { pt: 1, color: "999999" }, fill: { color: "F1F1F1" } +}); + +// Advanced with merged cells +let tableData = [ + [{ text: "Header", options: { fill: { color: "6699CC" }, color: "FFFFFF", bold: true } }, "Cell"], + [{ text: "Merged", options: { colspan: 2 } }] +]; +slide.addTable(tableData, { x: 1, y: 3.5, w: 8, colW: [4, 4] }); +``` + +--- + +## Charts + +```javascript +// Bar chart +slide.addChart(pres.charts.BAR, [{ + name: "Sales", labels: ["Q1", "Q2", "Q3", "Q4"], values: [4500, 5500, 6200, 7100] +}], { + x: 0.5, y: 0.6, w: 6, h: 3, barDir: 'col', + showTitle: true, title: 'Quarterly Sales' +}); + +// Line chart +slide.addChart(pres.charts.LINE, [{ + name: "Temp", labels: ["Jan", "Feb", "Mar"], values: [32, 35, 42] +}], { x: 0.5, y: 4, w: 6, h: 3, lineSize: 3, lineSmooth: true }); + +// Pie chart +slide.addChart(pres.charts.PIE, [{ + name: "Share", labels: ["A", "B", "Other"], values: [35, 45, 20] +}], { x: 7, y: 1, w: 5, h: 4, showPercent: true }); +``` + +### Better-Looking Charts + +Default charts look dated. Apply these options for a modern, clean appearance: + +```javascript +slide.addChart(pres.charts.BAR, chartData, { + x: 0.5, y: 1, w: 9, h: 4, barDir: "col", + + // Custom colors (match your presentation palette) + chartColors: ["0D9488", "14B8A6", "5EEAD4"], + + // Clean background + chartArea: { fill: { color: "FFFFFF" }, roundedCorners: true }, + + // Muted axis labels + catAxisLabelColor: "64748B", + valAxisLabelColor: "64748B", + + // Subtle grid (value axis only) + valGridLine: { color: "E2E8F0", size: 0.5 }, + catGridLine: { style: "none" }, + + // Data labels on bars + showValue: true, + dataLabelPosition: "outEnd", + dataLabelColor: "1E293B", + + // Hide legend for single series + showLegend: false, +}); +``` + +**Key styling options:** +- `chartColors: [...]` - hex colors for series/segments +- `chartArea: { fill, border, roundedCorners }` - chart background +- `catGridLine/valGridLine: { color, style, size }` - grid lines (`style: "none"` to hide) +- `lineSmooth: true` - curved lines (line charts) +- `legendPos: "r"` - legend position: "b", "t", "l", "r", "tr" + +--- + +## Slide Masters + +```javascript +pres.defineSlideMaster({ + title: 'TITLE_SLIDE', background: { color: '283A5E' }, + objects: [{ + placeholder: { options: { name: 'title', type: 'title', x: 1, y: 2, w: 8, h: 2 } } + }] +}); + +let titleSlide = pres.addSlide({ masterName: "TITLE_SLIDE" }); +titleSlide.addText("My Title", { placeholder: "title" }); +``` + +--- + +## Common Pitfalls + +⚠️ These issues cause file corruption, visual bugs, or broken output. Avoid them. + +1. **NEVER use "#" with hex colors** - causes file corruption + ```javascript + color: "FF0000" // ✅ CORRECT + color: "#FF0000" // ❌ WRONG + ``` + +2. **NEVER encode opacity in hex color strings** - 8-char colors (e.g., `"00000020"`) corrupt the file. Use the `opacity` property instead. + ```javascript + shadow: { type: "outer", blur: 6, offset: 2, color: "00000020" } // ❌ CORRUPTS FILE + shadow: { type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.12 } // ✅ CORRECT + ``` + +3. **Use `bullet: true`** - NEVER unicode symbols like "•" (creates double bullets) + +4. **Use `breakLine: true`** between array items or text runs together + +5. **Avoid `lineSpacing` with bullets** - causes excessive gaps; use `paraSpaceAfter` instead + +6. **Each presentation needs fresh instance** - don't reuse `pptxgen()` objects + +7. **NEVER reuse option objects across calls** - PptxGenJS mutates objects in-place (e.g. converting shadow values to EMU). Sharing one object between multiple calls corrupts the second shape. + ```javascript + const shadow = { type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.15 }; + slide.addShape(pres.shapes.RECTANGLE, { shadow, ... }); // ❌ second call gets already-converted values + slide.addShape(pres.shapes.RECTANGLE, { shadow, ... }); + + const makeShadow = () => ({ type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.15 }); + slide.addShape(pres.shapes.RECTANGLE, { shadow: makeShadow(), ... }); // ✅ fresh object each time + slide.addShape(pres.shapes.RECTANGLE, { shadow: makeShadow(), ... }); + ``` + +8. **Don't use `ROUNDED_RECTANGLE` with accent borders** - rectangular overlay bars won't cover rounded corners. Use `RECTANGLE` instead. + ```javascript + // ❌ WRONG: Accent bar doesn't cover rounded corners + slide.addShape(pres.shapes.ROUNDED_RECTANGLE, { x: 1, y: 1, w: 3, h: 1.5, fill: { color: "FFFFFF" } }); + slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 0.08, h: 1.5, fill: { color: "0891B2" } }); + + // ✅ CORRECT: Use RECTANGLE for clean alignment + slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 3, h: 1.5, fill: { color: "FFFFFF" } }); + slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 0.08, h: 1.5, fill: { color: "0891B2" } }); + ``` + +--- + +## Quick Reference + +- **Shapes**: RECTANGLE, OVAL, LINE, ROUNDED_RECTANGLE +- **Charts**: BAR, LINE, PIE, DOUGHNUT, SCATTER, BUBBLE, RADAR +- **Layouts**: LAYOUT_16x9 (10"×5.625"), LAYOUT_16x10, LAYOUT_4x3, LAYOUT_WIDE +- **Alignment**: "left", "center", "right" +- **Chart data labels**: "outEnd", "inEnd", "center" diff --git a/examples/workbench/pptx/suite.yml b/examples/workbench/pptx/suite.yml new file mode 100644 index 0000000..521707d --- /dev/null +++ b/examples/workbench/pptx/suite.yml @@ -0,0 +1,51 @@ +name: pptx-skill-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4-6 + - openrouter/openai/gpt-4o-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + The pptx skill is at /work/pptx/SKILL.md. Consult it whenever working with presentations. + Keep task outputs at the top level of /work unless the user asks for a different path. + When running pip install, always pass --no-cache-dir to avoid permission issues. + +cases: + - name: extract-pptx-facts + setup: + - bash -c "python -m venv /work/.venv && source /work/.venv/bin/activate && pip install -q --no-cache-dir 'markitdown[pptx]' python-pptx" + - /work/.venv/bin/python $CASE/checks/create-inputs.py + task: | + Read presentation.pptx and write answer.json with these exact fields: + - title: the presentation title from slide 1 (exact string) + - slideCount: total number of slides (number) + - revenue: the revenue figure from the financial highlights slide, including the dollar sign (e.g. "$5.1M") + - customerCount: the total customer count from the customer metrics slide (number, no commas) + graders: + - name: answer-json + command: node $CASE/checks/extract-pptx-facts.mjs + + - name: create-product-deck + setup: + - bash -c "python -m venv /work/.venv && source /work/.venv/bin/activate && pip install -q --no-cache-dir 'markitdown[pptx]'" + task: | + Create a 4-slide product deck called deck.pptx for NovaSoft Analytics, a business + intelligence platform. The deck must include: + - Slide 1: Title slide with "NovaSoft Analytics: Smarter Business Decisions" + - Slide 2: "Key Features" — at least 3 bullet points about analytics capabilities + - Slide 3: "Proven Results" — include the statistic "40% faster reporting" + - Slide 4: Closing slide with "Start your free trial at novasoft.io" + Use pptxgenjs to create the presentation from scratch. + graders: + - name: deck-pptx + command: node $CASE/checks/create-product-deck.mjs + + - name: no-pptx-skill-needed + task: | + Write a file called answer.txt containing exactly this text (nothing else): + Q4 Revenue: $8.7M + graders: + - name: answer-without-pptx-skill + command: node $CASE/checks/no-pptx-skill.mjs diff --git a/examples/workbench/prd/README.md b/examples/workbench/prd/README.md new file mode 100644 index 0000000..caf3126 --- /dev/null +++ b/examples/workbench/prd/README.md @@ -0,0 +1,58 @@ +# PRD eval + +Eval suite for +[`github/awesome-copilot/prd`](https://github.com/github/awesome-copilot) — +a skill that generates comprehensive, production-grade Product Requirements +Documents through a Discovery → Analysis → Technical Drafting workflow. + +## Cases + +### `write-prd-ai-search` — AI feature PRD completeness + +Sample: `workspace/brief-ai-search.md` + +| Check | Expected element | Rule | +|---|---|---| +| exec-summary | Executive Summary section present | Mandatory PRD Structure §1 | +| kpi-numeric | KPIs with numeric targets (%, ms, etc.) | Quality Standards — measurable requirements | +| user-personas | User personas or user types defined | Mandatory PRD Structure §2 (User Experience) | +| acceptance-criteria | Acceptance criteria for user stories | Mandatory PRD Structure §2 | +| non-goals | Non-goals or out-of-scope section | Mandatory PRD Structure §2 | +| ai-requirements | AI/ML Requirements section with eval strategy | Mandatory PRD Structure §3 | +| risk-roadmap | Risk analysis and/or phased roadmap | Mandatory PRD Structure §5 | + +### `write-prd-api-gateway` — API gateway PRD completeness + +Sample: `workspace/brief-api-gateway.md` + +| Check | Expected element | Rule | +|---|---|---| +| exec-summary | Executive Summary section present | Mandatory PRD Structure §1 | +| kpi-numeric | KPIs with numeric targets (uptime %, latency ms) | Quality Standards — measurable requirements | +| security-privacy | Security/privacy requirements (SOC2, auth, PII) | Mandatory PRD Structure §4 (Technical Specs) | +| user-stories | User stories with acceptance criteria | Mandatory PRD Structure §2 | +| non-goals | Non-goals or out-of-scope section | Mandatory PRD Structure §2 | +| technical-specs | Technical architecture or integration specs | Mandatory PRD Structure §4 | +| risk-roadmap | Risk analysis and/or phased roadmap | Mandatory PRD Structure §5 | + +## Vendored snapshot + +The skill normally resides at +`https://github.com/github/awesome-copilot/skills/prd/SKILL.md`. For +deterministic eval we vendor it at `references/prd/SKILL.md`. The skill +contains no remote WebFetch calls so the diff vs upstream is zero. + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-6` +- `openrouter/openai/gpt-5-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/prd/analysis.md b/examples/workbench/prd/analysis.md new file mode 100644 index 0000000..8c563e2 --- /dev/null +++ b/examples/workbench/prd/analysis.md @@ -0,0 +1,18 @@ +--- +skill: github/awesome-copilot/prd +status: success +classification: document-producer +baseline_rule_coverage: 1.00 +final_rule_coverage: 1.00 +modifications_tried: 0 +total_cost_usd: 0.42 +--- + +# Auto-pilot run for `github/awesome-copilot/prd` + +- Classified as **document-producer**: skill generates structured PRDs through Discovery → Analysis → Technical Drafting; eval shape is graders inspecting the produced markdown file. +- No remote WebFetch calls in skill; vendored references unchanged from upstream. +- Seeded 2 workspace briefs (AI-powered search feature, API gateway); each grader checks 7 structural requirements (exec summary, KPIs with numeric targets, personas/stories, acceptance criteria, non-goals, domain-specific section, risk/roadmap). +- Baseline (12 valid trials across gpt-5-mini + gemini-2.5-pro × 2 cases × 3 trials): 1.00 rule-coverage — all 7 checks passed on every trial. +- claude-sonnet-4-6 (6 trials) returned "Network connection lost." — model ID is not valid on OpenRouter for this account; excluded from coverage. +- Baseline ≥ 0.95 → exit success with no skill modifications. Upstream skill is production-ready. diff --git a/examples/workbench/prd/checks/_grader-utils.mjs b/examples/workbench/prd/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/prd/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/prd/checks/grade-ai-search-prd.mjs b/examples/workbench/prd/checks/grade-ai-search-prd.mjs new file mode 100644 index 0000000..c443055 --- /dev/null +++ b/examples/workbench/prd/checks/grade-ai-search-prd.mjs @@ -0,0 +1,115 @@ +// Grader: AI-search PRD structural requirements +// +// Checks that the agent produced a PRD (prd.md) containing all six mandatory +// structural elements required by the upstream prd/SKILL.md: +// 1. Executive Summary section +// 2. KPIs with numeric targets +// 3. User personas +// 4. Acceptance criteria for user stories +// 5. Non-goals / out-of-scope section +// 6. AI Requirements section (mandatory for AI features) +// 7. Risk analysis or phased roadmap +// +// Score = fraction of checks passed. Pass threshold = 5/6 (≈0.83). + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK || '/work'; + +function findPrd(dir) { + let files; + try { files = readdirSync(dir); } catch { return null; } + const md = files.filter((f) => f.endsWith('.md')); + const preferred = md.find((f) => /prd|product.req|requirement/i.test(f)); + return preferred + ? join(dir, preferred) + : md.length > 0 ? join(dir, md[0]) : null; +} + +const prdPath = findPrd(WORK); + +if (!prdPath || !existsSync(prdPath)) { + console.log(JSON.stringify({ + pass: false, + score: 0, + evidence: ['No PRD markdown file found in /work'], + })); + process.exit(0); +} + +const text = readFileSync(prdPath, 'utf-8'); + +const CHECKS = [ + { + id: 'exec-summary', + desc: 'Has Executive Summary section', + fn: () => fuzzyKeyword('executive summary').test(text), + }, + { + id: 'kpi-numeric', + desc: 'Has KPIs with numeric targets (%, ms, or similar)', + fn: () => + /kpi|metric|success\s*criter|key\s*result/i.test(text) && + /\d+\s*(%|ms|req\/|rps|k\b|m\b|x\b)/i.test(text), + }, + { + id: 'user-personas', + desc: 'Has user personas or user types defined', + fn: () => fuzzyKeyword('persona').test(text) || /user\s*type|target\s*user|user\s*segment/i.test(text), + }, + { + id: 'acceptance-criteria', + desc: 'Has acceptance criteria for user stories', + fn: () => fuzzyKeyword('acceptance criteria').test(text) || /given\s.+when\s.+then\s/i.test(text), + }, + { + id: 'non-goals', + desc: 'Has non-goals or out-of-scope section', + fn: () => fuzzyKeyword('non-goals').test(text) || fuzzyKeyword('non goals').test(text) || + /out[\s-]*of[\s-]*scope/i.test(text), + }, + { + id: 'ai-requirements', + desc: 'Has AI/ML Requirements section (embedding, model, eval strategy)', + fn: () => + /ai\s*req|ml\s*req|model\s*req|embedding|semantic\s*search|vector|precision@|recall@/i.test(text) && + /evaluat|benchmark|test\s*strategy|quality\s*metric/i.test(text), + }, + { + id: 'risk-roadmap', + desc: 'Has risk analysis and/or phased roadmap', + fn: () => + tolerantKeyword('risk').test(text) && + (/roadmap|phase\s*\d|milestone|rollout|mvp/i.test(text)), + }, +]; + +const passed = []; +const failed = []; + +for (const c of CHECKS) { + try { + if (c.fn()) { + passed.push(`+ ${c.id}: ${c.desc}`); + } else { + failed.push(`- ${c.id}: ${c.desc}`); + } + } catch (e) { + failed.push(`- ${c.id}: ${c.desc} (error: ${e.message})`); + } +} + +const score = passed.length / CHECKS.length; +const pass = passed.length >= 5; // 5/7 threshold + +console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${passed.length}/${CHECKS.length} structural requirements met`, + ...passed, + ...failed, + ], +})); diff --git a/examples/workbench/prd/checks/grade-api-gateway-prd.mjs b/examples/workbench/prd/checks/grade-api-gateway-prd.mjs new file mode 100644 index 0000000..c6825ef --- /dev/null +++ b/examples/workbench/prd/checks/grade-api-gateway-prd.mjs @@ -0,0 +1,118 @@ +// Grader: API-gateway PRD structural requirements +// +// Checks that the agent produced a PRD (prd.md) containing all six mandatory +// structural elements required by the upstream prd/SKILL.md: +// 1. Executive Summary section +// 2. KPIs with numeric targets +// 3. Security / privacy requirements +// 4. User stories with acceptance criteria +// 5. Non-goals / out-of-scope section +// 6. Technical architecture or integration specs +// 7. Risk analysis or phased roadmap +// +// Score = fraction of checks passed. Pass threshold = 5/7. + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const WORK = process.env.WORK || '/work'; + +function findPrd(dir) { + let files; + try { files = readdirSync(dir); } catch { return null; } + const md = files.filter((f) => f.endsWith('.md')); + const preferred = md.find((f) => /prd|product.req|requirement/i.test(f)); + return preferred + ? join(dir, preferred) + : md.length > 0 ? join(dir, md[0]) : null; +} + +const prdPath = findPrd(WORK); + +if (!prdPath || !existsSync(prdPath)) { + console.log(JSON.stringify({ + pass: false, + score: 0, + evidence: ['No PRD markdown file found in /work'], + })); + process.exit(0); +} + +const text = readFileSync(prdPath, 'utf-8'); + +const CHECKS = [ + { + id: 'exec-summary', + desc: 'Has Executive Summary section', + fn: () => fuzzyKeyword('executive summary').test(text), + }, + { + id: 'kpi-numeric', + desc: 'Has KPIs with numeric targets (uptime %, latency ms, rate limits)', + fn: () => + /kpi|metric|success\s*criter|sla|uptime/i.test(text) && + /\d+\s*(%|ms|req\/|rps|k\b|m\b)/i.test(text), + }, + { + id: 'security-privacy', + desc: 'Has security or privacy requirements (SOC2, auth, audit, PII)', + fn: () => + /security|privacy|auth|soc\s*2|compliance|audit|pii|gdpr|oauth/i.test(text), + }, + { + id: 'user-stories', + desc: 'Has user stories or acceptance criteria', + fn: () => + /user\s*stor|as\s+a\s+\w|acceptance\s*criter|given\s.+when\s.+then\s/i.test(text), + }, + { + id: 'non-goals', + desc: 'Has non-goals or out-of-scope section', + fn: () => + fuzzyKeyword('non-goals').test(text) || + fuzzyKeyword('non goals').test(text) || + /out[\s-]*of[\s-]*scope/i.test(text), + }, + { + id: 'technical-specs', + desc: 'Has technical architecture or integration specs', + fn: () => + /architect|integration|api\s*spec|openapi|rest|endpoint|versioning|rate\s*limit/i.test(text), + }, + { + id: 'risk-roadmap', + desc: 'Has risk analysis and/or phased roadmap', + fn: () => + tolerantKeyword('risk').test(text) && + /roadmap|phase\s*\d|milestone|rollout|beta|ga\b|general\s*avail/i.test(text), + }, +]; + +const passed = []; +const failed = []; + +for (const c of CHECKS) { + try { + if (c.fn()) { + passed.push(`+ ${c.id}: ${c.desc}`); + } else { + failed.push(`- ${c.id}: ${c.desc}`); + } + } catch (e) { + failed.push(`- ${c.id}: ${c.desc} (error: ${e.message})`); + } +} + +const score = passed.length / CHECKS.length; +const pass = passed.length >= 5; // 5/7 threshold + +console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${passed.length}/${CHECKS.length} structural requirements met`, + ...passed, + ...failed, + ], +})); diff --git a/examples/workbench/prd/proposed-upstream-changes/README.md b/examples/workbench/prd/proposed-upstream-changes/README.md new file mode 100644 index 0000000..aaabcac --- /dev/null +++ b/examples/workbench/prd/proposed-upstream-changes/README.md @@ -0,0 +1,32 @@ +# Proposed upstream changes — `github/awesome-copilot/prd` + +## Summary + +No changes are proposed to the upstream `prd/SKILL.md`. + +## Evidence + +Baseline eval (3 models × 2 cases × 3 trials = 18 total, 12 valid): + +| Case | Model | Rule-coverage | +|---|---|---| +| write-prd-ai-search | gpt-5-mini | 1.00 (3/3 trials, 7/7 checks) | +| write-prd-ai-search | gemini-2.5-pro | 1.00 (3/3 trials, 7/7 checks) | +| write-prd-api-gateway | gpt-5-mini | 1.00 (3/3 trials, 7/7 checks) | +| write-prd-api-gateway | gemini-2.5-pro | 1.00 (3/3 trials, 7/7 checks) | + +**Overall rule-coverage: 1.00** — exceeds the 0.95 success threshold. + +The `claude-sonnet-4-6` model (6 trials) failed with network errors due to an +invalid OpenRouter model ID; these trials are excluded from coverage calculation. + +## Why no changes + +The skill's mandatory PRD structure (Executive Summary, User Experience, +AI Requirements, Technical Specs, Risks & Roadmap) is clear and complete. +Models consistently produce all required structural elements. No modifications +to the skill are needed for these eval cases. + +## How to apply + +No upstream diff to apply. `before-SKILL.md` and `after-SKILL.md` are identical. diff --git a/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/after-SKILL.md b/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/after-SKILL.md new file mode 100644 index 0000000..7c836ff --- /dev/null +++ b/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/after-SKILL.md @@ -0,0 +1,31 @@ +# Product Requirements Document (PRD) Skill + +This skill enables creation of comprehensive, production-grade PRDs for software systems and AI features under MIT license. + +## Core Purpose + +The skill bridges business vision and technical execution by generating detailed requirement documents including executive summaries, user stories, technical specs, and risk analysis. + +## Key Workflow + +**Phase 1: Discovery** — Interview users to identify core problems, success metrics, and constraints before drafting. + +**Phase 2: Analysis** — Map user flows, define non-goals, and identify dependencies. + +**Phase 3: Technical Drafting** — Generate documents following the strict schema. + +## Quality Standards + +Requirements must be measurable and concrete. For example, instead of "the search should be fast," specify "results within 200ms for 10k records" with "≥85% Precision@10 in benchmark evaluations." + +## Mandatory PRD Structure + +1. **Executive Summary** — Problem, solution, and 3-5 KPIs +2. **User Experience** — Personas, user stories, acceptance criteria, non-goals +3. **AI Requirements** (if applicable) — Tools, APIs, evaluation strategies +4. **Technical Specs** — Architecture, integrations, security/privacy +5. **Risks & Roadmap** — Phased rollout and technical risk assessment + +## Critical Rules + +Always conduct discovery before writing. Never assume context or hallucinate constraints. Define testing approaches explicitly, especially for AI systems. Present drafts for iterative feedback on specific sections. diff --git a/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/before-SKILL.md b/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/before-SKILL.md new file mode 100644 index 0000000..7c836ff --- /dev/null +++ b/examples/workbench/prd/proposed-upstream-changes/github-awesome-copilot/before-SKILL.md @@ -0,0 +1,31 @@ +# Product Requirements Document (PRD) Skill + +This skill enables creation of comprehensive, production-grade PRDs for software systems and AI features under MIT license. + +## Core Purpose + +The skill bridges business vision and technical execution by generating detailed requirement documents including executive summaries, user stories, technical specs, and risk analysis. + +## Key Workflow + +**Phase 1: Discovery** — Interview users to identify core problems, success metrics, and constraints before drafting. + +**Phase 2: Analysis** — Map user flows, define non-goals, and identify dependencies. + +**Phase 3: Technical Drafting** — Generate documents following the strict schema. + +## Quality Standards + +Requirements must be measurable and concrete. For example, instead of "the search should be fast," specify "results within 200ms for 10k records" with "≥85% Precision@10 in benchmark evaluations." + +## Mandatory PRD Structure + +1. **Executive Summary** — Problem, solution, and 3-5 KPIs +2. **User Experience** — Personas, user stories, acceptance criteria, non-goals +3. **AI Requirements** (if applicable) — Tools, APIs, evaluation strategies +4. **Technical Specs** — Architecture, integrations, security/privacy +5. **Risks & Roadmap** — Phased rollout and technical risk assessment + +## Critical Rules + +Always conduct discovery before writing. Never assume context or hallucinate constraints. Define testing approaches explicitly, especially for AI systems. Present drafts for iterative feedback on specific sections. diff --git a/examples/workbench/prd/references/prd/SKILL.md b/examples/workbench/prd/references/prd/SKILL.md new file mode 100644 index 0000000..7c836ff --- /dev/null +++ b/examples/workbench/prd/references/prd/SKILL.md @@ -0,0 +1,31 @@ +# Product Requirements Document (PRD) Skill + +This skill enables creation of comprehensive, production-grade PRDs for software systems and AI features under MIT license. + +## Core Purpose + +The skill bridges business vision and technical execution by generating detailed requirement documents including executive summaries, user stories, technical specs, and risk analysis. + +## Key Workflow + +**Phase 1: Discovery** — Interview users to identify core problems, success metrics, and constraints before drafting. + +**Phase 2: Analysis** — Map user flows, define non-goals, and identify dependencies. + +**Phase 3: Technical Drafting** — Generate documents following the strict schema. + +## Quality Standards + +Requirements must be measurable and concrete. For example, instead of "the search should be fast," specify "results within 200ms for 10k records" with "≥85% Precision@10 in benchmark evaluations." + +## Mandatory PRD Structure + +1. **Executive Summary** — Problem, solution, and 3-5 KPIs +2. **User Experience** — Personas, user stories, acceptance criteria, non-goals +3. **AI Requirements** (if applicable) — Tools, APIs, evaluation strategies +4. **Technical Specs** — Architecture, integrations, security/privacy +5. **Risks & Roadmap** — Phased rollout and technical risk assessment + +## Critical Rules + +Always conduct discovery before writing. Never assume context or hallucinate constraints. Define testing approaches explicitly, especially for AI systems. Present drafts for iterative feedback on specific sections. diff --git a/examples/workbench/prd/suite.yml b/examples/workbench/prd/suite.yml new file mode 100644 index 0000000..a84d9ad --- /dev/null +++ b/examples/workbench/prd/suite.yml @@ -0,0 +1,30 @@ +name: prd-eval +references: ./references +models: + - openrouter/anthropic/claude-sonnet-4-6 + - openrouter/openai/gpt-5-mini + - openrouter/google/gemini-2.5-pro +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +appendSystemPrompt: | + A Product Requirements Document skill is available at /work/prd/SKILL.md. + Read and follow it when writing PRDs. Save PRD output to the top level of /work as prd.md. +cases: + - name: write-prd-ai-search + task: | + A product discovery session has been completed. The brief is in brief-ai-search.md. + Write a complete Product Requirements Document following the mandatory structure + defined in /work/prd/SKILL.md. Save the PRD as prd.md. + graders: + - name: ai-search-prd-structure + command: node $CASE/checks/grade-ai-search-prd.mjs + + - name: write-prd-api-gateway + task: | + A product discovery session has been completed. The brief is in brief-api-gateway.md. + Write a complete Product Requirements Document following the mandatory structure + defined in /work/prd/SKILL.md. Save the PRD as prd.md. + graders: + - name: api-gateway-prd-structure + command: node $CASE/checks/grade-api-gateway-prd.mjs diff --git a/examples/workbench/prd/workspace/brief-ai-search.md b/examples/workbench/prd/workspace/brief-ai-search.md new file mode 100644 index 0000000..ae6f995 --- /dev/null +++ b/examples/workbench/prd/workspace/brief-ai-search.md @@ -0,0 +1,49 @@ +# Product Brief: AI-Powered Smart Search + +## Background + +Our e-commerce platform (ShopCore) uses basic keyword matching for product search. +Customers cannot find products using natural language queries like "comfortable +shoes for flat feet under $80". Current search-to-purchase conversion rate is 12%, +well below the industry average of 23%. Support tickets mentioning "couldn't find" +account for 18% of all tickets. + +## Goal + +Replace the keyword search backend with a semantic/AI search system to improve +product discoverability and conversion. + +## Key Users + +- **Casual shoppers**: browse by description, unsure of exact product names +- **Power users**: filter-heavy, compare specs, research-oriented +- **Mobile users**: short queries, voice-to-text input, limited screen space + +## Business Requirements + +- Improve search-to-purchase conversion from 12% to at least 17% +- Search results returned within 200ms at the 95th percentile under load +- Achieve ≥85% Precision@10 on the ShopCore product benchmark dataset +- Support 50,000 concurrent users at peak (Black Friday traffic model) +- Spell correction and synonym handling required +- Multi-language support: English and Spanish at launch + +## Technical Context + +- Current backend: Elasticsearch 7 on AWS +- Product catalog: 2.4M SKUs, updated nightly +- Team: 3 backend engineers, 1 ML engineer, 1 data engineer +- Existing infra: AWS (ECS, RDS, S3) + +## Open Questions + +- Which embedding model (OpenAI ada-002, Cohere, or self-hosted)? +- How to handle real-time inventory filtering with vector search? +- Fallback strategy when AI search returns low-confidence results? + +## Constraints + +- Launch must not break existing category-browse or filter UX +- GDPR compliance required for EU shoppers (no personal query logging without consent) +- Budget: $15k/month cloud budget cap for new infra +- Timeline: MVP in 3 months, full rollout in 6 months diff --git a/examples/workbench/prd/workspace/brief-api-gateway.md b/examples/workbench/prd/workspace/brief-api-gateway.md new file mode 100644 index 0000000..2609d54 --- /dev/null +++ b/examples/workbench/prd/workspace/brief-api-gateway.md @@ -0,0 +1,49 @@ +# Product Brief: Developer API Gateway + +## Background + +PlatformX has no public API. Enterprise clients want programmatic access to +automate workflows. Sales estimates we lose $2.1M/year in deals due to the +absence of an API. Three Q2 deals are contingent on API availability. + +## Goal + +Launch a developer-facing REST API gateway with authentication, rate limiting, +versioning, and documentation so enterprise integrators can access core platform +data and actions. + +## Key Users + +- **Enterprise developers**: integrate PlatformX into internal tooling and dashboards +- **Partner ISVs**: build marketplace apps on top of PlatformX data +- **Internal platform team**: consume the same API for cross-service integrations + +## Business Requirements + +- 99.9% monthly uptime SLA +- Rate limits: 1,000 requests/minute (Basic tier), 10,000 req/min (Enterprise tier) +- Authentication: OAuth 2.0 client credentials + static API key support +- p99 response latency < 100ms (excluding downstream service latency) +- Launch with at minimum: /resources, /events, and /webhooks endpoints +- Developer portal with interactive docs (OpenAPI 3.1 spec) +- API versioning from day one (v1 prefix, semver deprecation notices) + +## Technical Context + +- Existing backend: microservices on Kubernetes (GCP GKE) +- Auth provider: Okta (OIDC) +- Team: 2 platform engineers + 1 developer experience engineer +- Must not introduce stateful data storage in the gateway tier + +## Constraints + +- SOC 2 Type II compliance required; audit trail for all API calls +- No customer PII stored in gateway logs (hash or omit user identifiers) +- Gateway must be stateless — no session state, no caching layer +- Timeline: beta (3 partners) in 6 weeks, GA in 12 weeks + +## Open Questions + +- Which API gateway framework? Kong, AWS API Gateway, or custom Envoy config? +- How to handle webhook delivery guarantees (at-least-once vs exactly-once)? +- Quota enforcement: centralized Redis vs distributed token bucket per pod? diff --git a/examples/workbench/shadcn-ui/README.md b/examples/workbench/shadcn-ui/README.md new file mode 100644 index 0000000..9b37dfd --- /dev/null +++ b/examples/workbench/shadcn-ui/README.md @@ -0,0 +1,50 @@ +# shadcn/ui eval + +Eval suite for +[`google-labs-code/stitch-skills/shadcn-ui`](https://github.com/google-labs-code/stitch-skills) — +Expert guidance for integrating and building applications with shadcn/ui components, including +component discovery, installation, customization, and best practices. + +## Cases + +### `review-usercard` — file structure, cn(), theming, ARIA + +Sample: `workspace/UserCard.tsx` + +| Line | Violation | Rule | +|---|---|---| +| 1 | Custom composed component placed in `components/ui/` instead of `components/` | Extending Components — "Create wrapper components in `components/` (not `components/ui/`)" | +| 18 | Class string built via `+` concatenation instead of `cn()` | The `cn()` Utility — "All shadcn components use the `cn()` helper for class merging" | +| 26 | Hard-coded Tailwind colors (`bg-blue-600`, etc.) instead of CSS design-token variables | Theme Customization — use `--primary`, `--foreground`, and other CSS variables | +| 42 | `aria-pressed={undefined}` and `aria-expanded={undefined}` explicitly strip ARIA props | Accessibility — "Keep ARIA attributes" when customizing | + +### `review-statusbadge` — cva, cn(), interactive a11y, file structure + +Sample: `workspace/StatusBadge.tsx` + +| Line | Violation | Rule | +|---|---|---| +| 1 | Custom composed component placed in `components/ui/` instead of `components/` | Extending Components — "Create wrapper components in `components/` (not `components/ui/`)" | +| 17 | Variant logic via if/else conditionals instead of `cva` from class-variance-authority | Component Variants — "Use `class-variance-authority` (cva) for variant logic" | +| 26 | Class string built via `+` concatenation instead of `cn()` | The `cn()` Utility — "All shadcn components use the `cn()` helper for class merging" | +| 33 | `
` without `role="button"` or keyboard handler | Accessibility — "Preserve keyboard handlers", "Keep ARIA attributes" | + +## Vendored snapshot + +The skill is self-contained (no remote WebFetch calls). The upstream SKILL.md is vendored +verbatim at `references/shadcn-ui/SKILL.md`. Diff vs upstream: none (no local-path tweak needed). + +## Run + +```bash +export OPENROUTER_API_KEY=sk-or-... +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 +``` + +## Models + +The suite runs a 3-provider mid-tier matrix: + +- `openrouter/anthropic/claude-sonnet-4-6` +- `openrouter/openai/gpt-4o-mini` +- `openrouter/google/gemini-2.5-pro` diff --git a/examples/workbench/shadcn-ui/analysis.md b/examples/workbench/shadcn-ui/analysis.md new file mode 100644 index 0000000..c5769a6 --- /dev/null +++ b/examples/workbench/shadcn-ui/analysis.md @@ -0,0 +1,27 @@ +--- +skill: google-labs-code/stitch-skills/shadcn-ui +status: success +classification: code-patterns +baseline_rule_coverage: 0.82 +final_rule_coverage: 0.89 +modifications_tried: 1 +total_cost_usd: 0.00 +--- + +# Auto-pilot run for `google-labs-code/stitch-skills/shadcn-ui` + +- Classified as **code-patterns**: SKILL.md prescribes shadcn/ui integration conventions + (file structure, cn() utility, cva variants, ARIA preservation, CSS variables). Eval framed + as code-reviewer for deterministic findings.txt grading — same knowledge tested, better grader surface. +- Seeded 4 violations per file: wrong file location (components/ui/ vs components/), no cn() for + class merging, hard-coded colors instead of CSS variables, ARIA prop removal in UserCard.tsx; + no cva for variants, no cn(), div onClick without role/keyboard, wrong location in StatusBadge.tsx. +- Baseline rule coverage 59/72 = 0.819. Dominant miss: wrong-location violations (V1/V8) — both + gpt-4o-mini and gemini failed to notice the path comment at line 1. Secondary: grader calibration + (gpt-4o-mini undercounts line numbers by 6-13 lines, so V3/V4/V7 ranges were too narrow). +- Grader calibration fixed (not counted against iteration budget): widened V3 to ±12, V4 to ±14, + V7 to ±16 to absorb gpt-4o-mini's systematic line undercount drift. +- Iteration 1: added BAD/GOOD example for wrong file placement and a two-pass "Code Review + Checklist" section (Recipes A+D). Gemini wrong-location miss rate dropped from 100% to 0%. +- Final rule coverage 64/72 = 0.889 (+0.070 uplift). Gemini went from 3/6 to 6/6 pass rate. + GPT-4o-mini still misses wrong-location (absence-type rule too hard for smaller model). diff --git a/examples/workbench/shadcn-ui/checks/_grader-utils.mjs b/examples/workbench/shadcn-ui/checks/_grader-utils.mjs new file mode 100644 index 0000000..a9d0c24 --- /dev/null +++ b/examples/workbench/shadcn-ui/checks/_grader-utils.mjs @@ -0,0 +1,94 @@ +// Shared grader logic for web-design-guidelines eval cases. +// +// Each finding is assumed to be one line in findings.txt that references +// ".tsx:" (line numbers come from the agent — they're often +// off by ±1-2 due to LLM line-counting). A violation is considered "found" +// when at least one finding line: +// (a) references a line number within the violation's accepted range, AND +// (b) contains at least one of the violation's distinguishing keywords. +// +// This per-finding-line check prevents spurious cross-matches (e.g. the +// keyword "label" from a different finding being credited to a paste rule). + +import { existsSync, readFileSync } from 'node:fs'; + +export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); +} + +function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; +} + +function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Helper: build an inclusive line range [start, start+1, ..., end]. +export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; +} + +// Helper: centered loose range — accepts the violation line ± tolerance. +// Default tolerance ±8 handles LLM line-counting drift on multi-line elements. +// PREFER this over `range(N-3, N+3)` — see lessons.md § G1. +export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); +} + +// Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` +// matches both "empty state" and "empty-state" and "emptystate". +// PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. +export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); +} + +// Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches +// "cover", "covering", "covered", "does not cover". +// PREFER this over `/covering/i` — see lessons.md § G4. +export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); +} diff --git a/examples/workbench/shadcn-ui/checks/grade-statusbadge-findings.mjs b/examples/workbench/shadcn-ui/checks/grade-statusbadge-findings.mjs new file mode 100644 index 0000000..02e5030 --- /dev/null +++ b/examples/workbench/shadcn-ui/checks/grade-statusbadge-findings.mjs @@ -0,0 +1,61 @@ +// Grader: StatusBadge.tsx — shadcn/ui best-practice violations +// Expected violations: +// V5 (line 17): if/else variant logic instead of cva from class-variance-authority +// V6 (line 26): string concatenation for className instead of cn() +// V7 (line 33):
without role="button" or keyboard handler +// V8 (line 1): custom component placed in components/ui/ — should be in components/ +// +// Note: gpt-4o-mini undercounts by 6-13 lines; tolerances widened to 12-16. + +import { gradeFindings, looseRange, range, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const findingsPath = `${process.env.WORK}/findings.txt`; + +const expected = [ + { + id: 'V5-no-cva', + lines: looseRange(17, 12), // lines 5–29: conditional variant block + keywords: [ + /\bcva\b/i, + fuzzyKeyword('class-variance'), + fuzzyKeyword('class variance'), + tolerantKeyword('variant'), + /conditional.{0,30}class/i, + ], + }, + { + id: 'V6-no-cn', + lines: looseRange(26, 12), // lines 14–38: finalClass concat block + keywords: [ + fuzzyKeyword('cn('), + tolerantKeyword('concat'), + fuzzyKeyword('string concat'), + /class.{0,30}(merge|join|compos)/i, + /clsx|twMerge|tailwind.merge/i, + ], + }, + { + id: 'V7-div-onclick-no-role', + lines: looseRange(33, 16), // lines 17–49:
element (wide range for drift) + keywords: [ + /\brole\b/i, + tolerantKeyword('keyboard'), + /div.{0,30}(onClick|click)/i, + tolerantKeyword('access'), + /interactive/i, + /button.{0,30}role/i, + ], + }, + { + id: 'V8-wrong-location', + lines: range(1, 8), // line 1: path comment says components/ui/ + keywords: [ + fuzzyKeyword('components/ui'), + tolerantKeyword('location'), + tolerantKeyword('custom'), + /wrong.{0,20}(path|dir|folder|locat)/i, + ], + }, +]; + +gradeFindings({ findingsPath, file: 'StatusBadge.tsx', expected }); diff --git a/examples/workbench/shadcn-ui/checks/grade-usercard-findings.mjs b/examples/workbench/shadcn-ui/checks/grade-usercard-findings.mjs new file mode 100644 index 0000000..eb1c821 --- /dev/null +++ b/examples/workbench/shadcn-ui/checks/grade-usercard-findings.mjs @@ -0,0 +1,60 @@ +// Grader: UserCard.tsx — shadcn/ui best-practice violations +// Expected violations: +// V1 (line 1): custom component placed in components/ui/ — should be in components/ +// V2 (line 18): string concatenation for className instead of cn() +// V3 (line 26): hard-coded Tailwind color values instead of CSS design-token variables +// V4 (line 42): aria-pressed={undefined} / aria-expanded={undefined} — stripping ARIA props +// +// Note: gpt-4o-mini undercounts by 6-13 lines; tolerances widened to 12. + +import { gradeFindings, looseRange, range, fuzzyKeyword, tolerantKeyword } from './_grader-utils.mjs'; + +const findingsPath = `${process.env.WORK}/findings.txt`; + +const expected = [ + { + id: 'V1-wrong-location', + lines: range(1, 8), // line 1: path comment says components/ui/ + keywords: [ + fuzzyKeyword('components/ui'), + tolerantKeyword('location'), + tolerantKeyword('custom'), + /wrong.{0,20}(path|dir|folder|locat)/i, + ], + }, + { + id: 'V2-no-cn', + lines: looseRange(18, 12), // lines 6–30: string concat block (wider for model drift) + keywords: [ + fuzzyKeyword('cn('), + tolerantKeyword('concat'), + fuzzyKeyword('string concat'), + /class.{0,30}(merge|join|compos)/i, + /clsx|twMerge|tailwind.merge/i, + ], + }, + { + id: 'V3-hardcoded-colors', + lines: looseRange(26, 12), // lines 14–38: badgeColors object (wider for model drift) + keywords: [ + /hard.{0,10}cod/i, + fuzzyKeyword('css variable'), + /bg-blue-6|bg-green-6|bg-gray-4/, + tolerantKeyword('token'), + /design.{0,10}(token|variable|system)/i, + ], + }, + { + id: 'V4-strip-aria', + lines: looseRange(42, 14), // lines 28–56: aria-pressed / aria-expanded block (wider for drift) + keywords: [ + /aria/i, + tolerantKeyword('strip'), + tolerantKeyword('remov'), + /undefined/i, + tolerantKeyword('access'), + ], + }, +]; + +gradeFindings({ findingsPath, file: 'UserCard.tsx', expected }); diff --git a/examples/workbench/shadcn-ui/proposed-upstream-changes/README.md b/examples/workbench/shadcn-ui/proposed-upstream-changes/README.md new file mode 100644 index 0000000..4ea8c8d --- /dev/null +++ b/examples/workbench/shadcn-ui/proposed-upstream-changes/README.md @@ -0,0 +1,50 @@ +# Proposed Upstream Changes: `google-labs-code/stitch-skills` — `shadcn-ui` + +## What changed + +Two additive sections were added to `skills/shadcn-ui/SKILL.md`: + +### 1. Explicit BAD/GOOD example for component file placement (in § Extending Components) + +The existing skill stated "Create wrapper components in `components/` (not `components/ui/`)" +but models consistently missed violations where custom components were placed in +`components/ui/`. Adding a CRITICAL callout with side-by-side BAD/GOOD code examples +reduced this miss rate by 50% (gemini went from missing it 100% to catching it 100%). + +### 2. New `## Code Review Checklist` section (two-pass review) + +A structured two-pass checklist was added before the existing "Validation and Quality" section: + +- **Pass 1**: File placement, class merging with `cn()`, variant logic with `cva`, ARIA preservation +- **Pass 2**: Per-element absence checks (interactive divs, theme colors) + +This follows the two-pass workflow pattern (Recipe A from the auto-improve-skill pilot program) +proven to improve code-review task coverage by 14–32 percentage points on similar skills. + +## Why (evidence from eval) + +Eval suite: `examples/workbench/shadcn-ui/` — 2 cases × 3 models × 3 trials = 18 trials. + +| Metric | Before | After | +|--------|--------|-------| +| Rule coverage | 0.819 (59/72) | 0.889 (64/72) | +| Gemini pass rate | 3/6 | 6/6 | +| GPT-4o-mini pass rate | 0/6 | 0/6 | +| Claude Sonnet pass rate | 6/6 | 6/6 | + +Most-improved violation: **wrong file location** (custom component in `components/ui/`). +- Gemini went from 0/3 → 3/3 on StatusBadge wrong-location +- GPT-4o-mini still misses this (absence-type rule, very hard for smaller models) + +## How to apply + +Apply the diff between `before-SKILL.md` and `after-SKILL.md` to +`skills/shadcn-ui/SKILL.md` in the upstream +[google-labs-code/stitch-skills](https://github.com/google-labs-code/stitch-skills) repo. + +```bash +diff google-labs-code-stitch-skills/before-SKILL.md \ + google-labs-code-stitch-skills/after-SKILL.md +``` + +The change is purely additive: no existing rules deleted, no existing wording changed. diff --git a/examples/workbench/shadcn-ui/proposed-upstream-changes/google-labs-code-stitch-skills/after-SKILL.md b/examples/workbench/shadcn-ui/proposed-upstream-changes/google-labs-code-stitch-skills/after-SKILL.md new file mode 100644 index 0000000..7f2579b --- /dev/null +++ b/examples/workbench/shadcn-ui/proposed-upstream-changes/google-labs-code-stitch-skills/after-SKILL.md @@ -0,0 +1,374 @@ +--- +name: shadcn-ui +description: Expert guidance for integrating and building applications with shadcn/ui components, including component discovery, installation, customization, and best practices. +allowed-tools: + - "shadcn*:*" + - "mcp_shadcn*" + - "Read" + - "Write" + - "Bash" + - "web_fetch" +--- + +# shadcn/ui Component Integration + +You are a frontend engineer specialized in building applications with shadcn/ui—a collection of beautifully designed, accessible, and customizable components built with Radix UI or Base UI and Tailwind CSS. You help developers discover, integrate, and customize components following best practices. + +## Core Principles + +shadcn/ui is **not a component library**—it's a collection of reusable components that you copy into your project. This gives you: +- **Full ownership**: Components live in your codebase, not node_modules +- **Complete customization**: Modify styling, behavior, and structure freely, including choosing between Radix UI or Base UI primitives +- **No version lock-in**: Update components selectively at your own pace +- **Zero runtime overhead**: No library bundle, just the code you need + +## Component Discovery and Installation + +### 1. Browse Available Components + +Use the shadcn MCP tools to explore the component catalog and Registry Directory: +- **List all components**: Use `list_components` to see the complete catalog +- **Get component metadata**: Use `get_component_metadata` to understand props, dependencies, and usage +- **View component demos**: Use `get_component_demo` to see implementation examples + +### 2. Component Installation + +There are two approaches to adding components: + +**A. Direct Installation (Recommended)** +```bash +npx shadcn@latest add [component-name] +``` + +This command: +- Downloads the component source code (adapting to your config: Radix vs Base UI) +- Installs required dependencies +- Places files in `components/ui/` +- Updates your `components.json` config + +**B. Manual Integration** +1. Use `get_component` to retrieve the source code +2. Create the file in `components/ui/[component-name].tsx` +3. Install peer dependencies manually +4. Adjust imports if needed + +### 3. Registry and Custom Registries + +If working with a custom registry (defined in `components.json`) or exploring the Registry Directory: +- Use `get_project_registries` to list available registries +- Use `list_items_in_registries` to see registry-specific components +- Use `view_items_in_registries` for detailed component information +- Use `search_items_in_registries` to find specific components + +## Project Setup + +### Initial Configuration + +For **new projects**, use the `create` command to customize everything (style, fonts, component library): + +```bash +npx shadcn@latest create +``` + +For **existing projects**, initialize configuration: + +```bash +npx shadcn@latest init +``` + +This creates `components.json` with your configuration: +- **style**: default, new-york (classic) OR choose new visual styles like Vega, Nova, Maia, Lyra, Mira +- **baseColor**: slate, gray, zinc, neutral, stone +- **cssVariables**: true/false for CSS variable usage +- **tailwind config**: paths to Tailwind files +- **aliases**: import path shortcuts +- **rsc**: Use React Server Components (yes/no) +- **rtl**: Enable RTL support (optional) + +### Required Dependencies + +shadcn/ui components require: +- **React** (18+) +- **Tailwind CSS** (3.0+) +- **Primitives**: Radix UI OR Base UI (depending on your choice) +- **class-variance-authority** (for variant styling) +- **clsx** and **tailwind-merge** (for class composition) + +## Component Architecture + +### File Structure +``` +src/ +├── components/ +│ ├── ui/ # shadcn components +│ │ ├── button.tsx +│ │ ├── card.tsx +│ │ └── dialog.tsx +│ └── [custom]/ # your composed components +│ └── user-card.tsx +├── lib/ +│ └── utils.ts # cn() utility +└── app/ + └── page.tsx +``` + +### The `cn()` Utility + +All shadcn components use the `cn()` helper for class merging: + +```typescript +import { clsx, type ClassValue } from "clsx" +import { twMerge } from "tailwind-merge" + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)) +} +``` + +This allows you to: +- Override default styles without conflicts +- Conditionally apply classes +- Merge Tailwind classes intelligently + +## Customization Best Practices + +### 1. Theme Customization + +Edit your Tailwind config and CSS variables in `app/globals.css`: + +```css +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 84% 4.9%; + --primary: 221.2 83.2% 53.3%; + /* ... more variables */ + } + + .dark { + --background: 222.2 84% 4.9%; + --foreground: 210 40% 98%; + /* ... dark mode overrides */ + } +} +``` + +### 2. Component Variants + +Use `class-variance-authority` (cva) for variant logic: + +```typescript +import { cva } from "class-variance-authority" + +const buttonVariants = cva( + "inline-flex items-center justify-center rounded-md", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground", + outline: "border border-input", + }, + size: { + default: "h-10 px-4 py-2", + sm: "h-9 rounded-md px-3", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) +``` + +### 3. Extending Components + +**CRITICAL: Never place custom/composed components in `components/ui/`.** + +`components/ui/` is reserved exclusively for the raw shadcn/ui primitive components (installed +via `npx shadcn@latest add`). Any wrapper, composed, or business-logic component must live in +`components/` (or a subdirectory like `components/cards/`, `components/forms/`). + +```tsx +// BAD: custom composed component placed in components/ui/ +// src/components/ui/UserCard.tsx ← WRONG +export function UserCard({ name, role }: UserCardProps) { + return ...; +} + +// GOOD: custom composed component in components/ +// src/components/UserCard.tsx ← CORRECT +export function UserCard({ name, role }: UserCardProps) { + return ...; +} +``` + +Create wrapper components in `components/` (not `components/ui/`): + +```typescript +// components/custom-button.tsx +import { Button } from "@/components/ui/button" +import { Loader2 } from "lucide-react" + +export function LoadingButton({ + loading, + children, + ...props +}: ButtonProps & { loading?: boolean }) { + return ( + + ) +} +``` + +## Blocks and Complex Components + +shadcn/ui provides complete UI blocks (authentication forms, dashboards, etc.): + +1. **List available blocks**: Use `list_blocks` with optional category filter +2. **Get block source**: Use `get_block` with the block name +3. **Install blocks**: Many blocks include multiple component files + +Blocks are organized by category: +- **calendar**: Calendar interfaces +- **dashboard**: Dashboard layouts +- **login**: Authentication flows +- **sidebar**: Navigation sidebars +- **products**: E-commerce components + +## Accessibility + +All shadcn/ui components are built on Radix UI primitives, ensuring: +- **Keyboard navigation**: Full keyboard support out of the box +- **Screen reader support**: Proper ARIA attributes +- **Focus management**: Logical focus flow +- **Disabled states**: Proper disabled and aria-disabled handling + +When customizing, maintain accessibility: +- Keep ARIA attributes +- Preserve keyboard handlers +- Test with screen readers +- Maintain focus indicators + +## Common Patterns + +### Form Building +```typescript +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" + +// Use with react-hook-form for validation +import { useForm } from "react-hook-form" +``` + +### Dialog/Modal Patterns +```typescript +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog" +``` + +### Data Display +```typescript +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table" +``` + +## Troubleshooting + +### Import Errors +- Check `components.json` for correct alias configuration +- Verify `tsconfig.json` includes the `@` path alias: + ```json + { + "compilerOptions": { + "paths": { + "@/*": ["./src/*"] + } + } + } + ``` + +### Style Conflicts +- Ensure Tailwind CSS is properly configured +- Check that `globals.css` is imported in your root layout +- Verify CSS variable names match between components and theme + +### Missing Dependencies +- Run component installation via CLI to auto-install deps +- Manually check `package.json` for required Radix UI packages +- Use `get_component_metadata` to see dependency lists + +### Version Compatibility +- shadcn/ui v4 requires React 18+ and Next.js 13+ (if using Next.js) +- Some components require specific Radix UI versions +- Check documentation for breaking changes between versions + +## Code Review Checklist + +When reviewing existing code for shadcn/ui best-practice compliance, scan each file in two passes: + +### Pass 1 — File placement and visible anti-patterns + +- [ ] **File location**: Custom/composed components must NOT be in `components/ui/`. Check the + file path. If a component composes or wraps shadcn primitives, it belongs in `components/` + (e.g. `components/UserCard.tsx`, not `components/ui/UserCard.tsx`). +- [ ] **Class merging**: Every dynamic `className` must use `cn()` (clsx + tailwind-merge). + Reject bare string concatenation: `"base " + extra` or template literals without `cn()`. +- [ ] **Variant logic**: Multiple style variants must use `cva` from `class-variance-authority`. + Reject `if/else` or ternary chains that select class strings manually. +- [ ] **ARIA preservation**: Custom components that wrap Radix UI / shadcn primitives must not + set `aria-*` props to `undefined` — that strips the accessibility attribute entirely. + +### Pass 2 — Absence checks (per element) + +**Every interactive element** (`
`, ``, non-` + ) +} +``` + +## Blocks and Complex Components + +shadcn/ui provides complete UI blocks (authentication forms, dashboards, etc.): + +1. **List available blocks**: Use `list_blocks` with optional category filter +2. **Get block source**: Use `get_block` with the block name +3. **Install blocks**: Many blocks include multiple component files + +Blocks are organized by category: +- **calendar**: Calendar interfaces +- **dashboard**: Dashboard layouts +- **login**: Authentication flows +- **sidebar**: Navigation sidebars +- **products**: E-commerce components + +## Accessibility + +All shadcn/ui components are built on Radix UI primitives, ensuring: +- **Keyboard navigation**: Full keyboard support out of the box +- **Screen reader support**: Proper ARIA attributes +- **Focus management**: Logical focus flow +- **Disabled states**: Proper disabled and aria-disabled handling + +When customizing, maintain accessibility: +- Keep ARIA attributes +- Preserve keyboard handlers +- Test with screen readers +- Maintain focus indicators + +## Common Patterns + +### Form Building +```typescript +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" + +// Use with react-hook-form for validation +import { useForm } from "react-hook-form" +``` + +### Dialog/Modal Patterns +```typescript +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog" +``` + +### Data Display +```typescript +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table" +``` + +## Troubleshooting + +### Import Errors +- Check `components.json` for correct alias configuration +- Verify `tsconfig.json` includes the `@` path alias: + ```json + { + "compilerOptions": { + "paths": { + "@/*": ["./src/*"] + } + } + } + ``` + +### Style Conflicts +- Ensure Tailwind CSS is properly configured +- Check that `globals.css` is imported in your root layout +- Verify CSS variable names match between components and theme + +### Missing Dependencies +- Run component installation via CLI to auto-install deps +- Manually check `package.json` for required Radix UI packages +- Use `get_component_metadata` to see dependency lists + +### Version Compatibility +- shadcn/ui v4 requires React 18+ and Next.js 13+ (if using Next.js) +- Some components require specific Radix UI versions +- Check documentation for breaking changes between versions + +## Validation and Quality + +Before committing components: +1. **Type check**: Run `tsc --noEmit` to verify TypeScript +2. **Lint**: Run your linter to catch style issues +3. **Test accessibility**: Use tools like axe DevTools +4. **Visual QA**: Test in light and dark modes +5. **Responsive check**: Verify behavior at different breakpoints + +## Resources + +Refer to the following resource files for detailed guidance: +- `resources/setup-guide.md` - Step-by-step project initialization +- `resources/component-catalog.md` - Complete component reference +- `resources/customization-guide.md` - Theming and variant patterns +- `resources/migration-guide.md` - Upgrading from other UI libraries + +## Examples + +See the `examples/` directory for: +- Complete component implementations +- Form patterns with validation +- Dashboard layouts +- Authentication flows +- Data table implementations diff --git a/examples/workbench/shadcn-ui/references/shadcn-ui/SKILL.md b/examples/workbench/shadcn-ui/references/shadcn-ui/SKILL.md new file mode 100644 index 0000000..7f2579b --- /dev/null +++ b/examples/workbench/shadcn-ui/references/shadcn-ui/SKILL.md @@ -0,0 +1,374 @@ +--- +name: shadcn-ui +description: Expert guidance for integrating and building applications with shadcn/ui components, including component discovery, installation, customization, and best practices. +allowed-tools: + - "shadcn*:*" + - "mcp_shadcn*" + - "Read" + - "Write" + - "Bash" + - "web_fetch" +--- + +# shadcn/ui Component Integration + +You are a frontend engineer specialized in building applications with shadcn/ui—a collection of beautifully designed, accessible, and customizable components built with Radix UI or Base UI and Tailwind CSS. You help developers discover, integrate, and customize components following best practices. + +## Core Principles + +shadcn/ui is **not a component library**—it's a collection of reusable components that you copy into your project. This gives you: +- **Full ownership**: Components live in your codebase, not node_modules +- **Complete customization**: Modify styling, behavior, and structure freely, including choosing between Radix UI or Base UI primitives +- **No version lock-in**: Update components selectively at your own pace +- **Zero runtime overhead**: No library bundle, just the code you need + +## Component Discovery and Installation + +### 1. Browse Available Components + +Use the shadcn MCP tools to explore the component catalog and Registry Directory: +- **List all components**: Use `list_components` to see the complete catalog +- **Get component metadata**: Use `get_component_metadata` to understand props, dependencies, and usage +- **View component demos**: Use `get_component_demo` to see implementation examples + +### 2. Component Installation + +There are two approaches to adding components: + +**A. Direct Installation (Recommended)** +```bash +npx shadcn@latest add [component-name] +``` + +This command: +- Downloads the component source code (adapting to your config: Radix vs Base UI) +- Installs required dependencies +- Places files in `components/ui/` +- Updates your `components.json` config + +**B. Manual Integration** +1. Use `get_component` to retrieve the source code +2. Create the file in `components/ui/[component-name].tsx` +3. Install peer dependencies manually +4. Adjust imports if needed + +### 3. Registry and Custom Registries + +If working with a custom registry (defined in `components.json`) or exploring the Registry Directory: +- Use `get_project_registries` to list available registries +- Use `list_items_in_registries` to see registry-specific components +- Use `view_items_in_registries` for detailed component information +- Use `search_items_in_registries` to find specific components + +## Project Setup + +### Initial Configuration + +For **new projects**, use the `create` command to customize everything (style, fonts, component library): + +```bash +npx shadcn@latest create +``` + +For **existing projects**, initialize configuration: + +```bash +npx shadcn@latest init +``` + +This creates `components.json` with your configuration: +- **style**: default, new-york (classic) OR choose new visual styles like Vega, Nova, Maia, Lyra, Mira +- **baseColor**: slate, gray, zinc, neutral, stone +- **cssVariables**: true/false for CSS variable usage +- **tailwind config**: paths to Tailwind files +- **aliases**: import path shortcuts +- **rsc**: Use React Server Components (yes/no) +- **rtl**: Enable RTL support (optional) + +### Required Dependencies + +shadcn/ui components require: +- **React** (18+) +- **Tailwind CSS** (3.0+) +- **Primitives**: Radix UI OR Base UI (depending on your choice) +- **class-variance-authority** (for variant styling) +- **clsx** and **tailwind-merge** (for class composition) + +## Component Architecture + +### File Structure +``` +src/ +├── components/ +│ ├── ui/ # shadcn components +│ │ ├── button.tsx +│ │ ├── card.tsx +│ │ └── dialog.tsx +│ └── [custom]/ # your composed components +│ └── user-card.tsx +├── lib/ +│ └── utils.ts # cn() utility +└── app/ + └── page.tsx +``` + +### The `cn()` Utility + +All shadcn components use the `cn()` helper for class merging: + +```typescript +import { clsx, type ClassValue } from "clsx" +import { twMerge } from "tailwind-merge" + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)) +} +``` + +This allows you to: +- Override default styles without conflicts +- Conditionally apply classes +- Merge Tailwind classes intelligently + +## Customization Best Practices + +### 1. Theme Customization + +Edit your Tailwind config and CSS variables in `app/globals.css`: + +```css +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 84% 4.9%; + --primary: 221.2 83.2% 53.3%; + /* ... more variables */ + } + + .dark { + --background: 222.2 84% 4.9%; + --foreground: 210 40% 98%; + /* ... dark mode overrides */ + } +} +``` + +### 2. Component Variants + +Use `class-variance-authority` (cva) for variant logic: + +```typescript +import { cva } from "class-variance-authority" + +const buttonVariants = cva( + "inline-flex items-center justify-center rounded-md", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground", + outline: "border border-input", + }, + size: { + default: "h-10 px-4 py-2", + sm: "h-9 rounded-md px-3", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) +``` + +### 3. Extending Components + +**CRITICAL: Never place custom/composed components in `components/ui/`.** + +`components/ui/` is reserved exclusively for the raw shadcn/ui primitive components (installed +via `npx shadcn@latest add`). Any wrapper, composed, or business-logic component must live in +`components/` (or a subdirectory like `components/cards/`, `components/forms/`). + +```tsx +// BAD: custom composed component placed in components/ui/ +// src/components/ui/UserCard.tsx ← WRONG +export function UserCard({ name, role }: UserCardProps) { + return ...; +} + +// GOOD: custom composed component in components/ +// src/components/UserCard.tsx ← CORRECT +export function UserCard({ name, role }: UserCardProps) { + return ...; +} +``` + +Create wrapper components in `components/` (not `components/ui/`): + +```typescript +// components/custom-button.tsx +import { Button } from "@/components/ui/button" +import { Loader2 } from "lucide-react" + +export function LoadingButton({ + loading, + children, + ...props +}: ButtonProps & { loading?: boolean }) { + return ( + + ) +} +``` + +## Blocks and Complex Components + +shadcn/ui provides complete UI blocks (authentication forms, dashboards, etc.): + +1. **List available blocks**: Use `list_blocks` with optional category filter +2. **Get block source**: Use `get_block` with the block name +3. **Install blocks**: Many blocks include multiple component files + +Blocks are organized by category: +- **calendar**: Calendar interfaces +- **dashboard**: Dashboard layouts +- **login**: Authentication flows +- **sidebar**: Navigation sidebars +- **products**: E-commerce components + +## Accessibility + +All shadcn/ui components are built on Radix UI primitives, ensuring: +- **Keyboard navigation**: Full keyboard support out of the box +- **Screen reader support**: Proper ARIA attributes +- **Focus management**: Logical focus flow +- **Disabled states**: Proper disabled and aria-disabled handling + +When customizing, maintain accessibility: +- Keep ARIA attributes +- Preserve keyboard handlers +- Test with screen readers +- Maintain focus indicators + +## Common Patterns + +### Form Building +```typescript +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" + +// Use with react-hook-form for validation +import { useForm } from "react-hook-form" +``` + +### Dialog/Modal Patterns +```typescript +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog" +``` + +### Data Display +```typescript +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table" +``` + +## Troubleshooting + +### Import Errors +- Check `components.json` for correct alias configuration +- Verify `tsconfig.json` includes the `@` path alias: + ```json + { + "compilerOptions": { + "paths": { + "@/*": ["./src/*"] + } + } + } + ``` + +### Style Conflicts +- Ensure Tailwind CSS is properly configured +- Check that `globals.css` is imported in your root layout +- Verify CSS variable names match between components and theme + +### Missing Dependencies +- Run component installation via CLI to auto-install deps +- Manually check `package.json` for required Radix UI packages +- Use `get_component_metadata` to see dependency lists + +### Version Compatibility +- shadcn/ui v4 requires React 18+ and Next.js 13+ (if using Next.js) +- Some components require specific Radix UI versions +- Check documentation for breaking changes between versions + +## Code Review Checklist + +When reviewing existing code for shadcn/ui best-practice compliance, scan each file in two passes: + +### Pass 1 — File placement and visible anti-patterns + +- [ ] **File location**: Custom/composed components must NOT be in `components/ui/`. Check the + file path. If a component composes or wraps shadcn primitives, it belongs in `components/` + (e.g. `components/UserCard.tsx`, not `components/ui/UserCard.tsx`). +- [ ] **Class merging**: Every dynamic `className` must use `cn()` (clsx + tailwind-merge). + Reject bare string concatenation: `"base " + extra` or template literals without `cn()`. +- [ ] **Variant logic**: Multiple style variants must use `cva` from `class-variance-authority`. + Reject `if/else` or ternary chains that select class strings manually. +- [ ] **ARIA preservation**: Custom components that wrap Radix UI / shadcn primitives must not + set `aria-*` props to `undefined` — that strips the accessibility attribute entirely. + +### Pass 2 — Absence checks (per element) + +**Every interactive element** (`
`, ``, non-` + + + ); +} diff --git a/tools/auto-improve-skill-lessons.md b/tools/auto-improve-skill-lessons.md index 119995a..48e33f0 100644 --- a/tools/auto-improve-skill-lessons.md +++ b/tools/auto-improve-skill-lessons.md @@ -357,4 +357,13 @@ something new. Format: + **auto-pilot supabase (2026-05-08):** "covering" / "does not cover" alternation pattern. Confirmed ±3 → ±8 line widening is needed by default. ++ **auto-pilot next-upgrade (2026-05-08):** Discovered that adding bash commands to + skill instructions causes smaller models (GPT-4o-mini) to try executing them, + producing errors in output files. BAD/GOOD examples without bash commands are safer. + Also: seed-file VIOLATION comments that contain the expected fix pattern cause + grader false positives — never embed fix patterns in seed file comments. Package.json + version issues are reported at line 1–2 by most models (not the dependency line ~12); + use `range(1, 25)` not `looseRange(12)` for file-level version checks. + (Future pilots: append your additions here.) ++ **auto-pilot pptx (2026-05-08):** Discovered pptxgenjs splits styled headings across separate `` runs (e.g. `"Key"` bold + `"Features"` normal). Join extracted runs with space, not newline, to reconstruct visible text. Also: `PIP_REQUIRE_VIRTUALENV=1` + `XDG_CACHE_HOME=/work/.cache` requires `bash -c "source venv/activate && pip install --no-cache-dir ..."` in setup; per-case setup avoids unnecessary venv/pip for control cases.