diff --git a/.gitignore b/.gitignore index 087e5ce..6e09010 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,6 @@ cli-commands.json tools.json tasks.json .worktrees/ + +# Auto-improve-skill wrapper run logs +examples/workbench/*/.run.log diff --git a/tools/auto-improve-skill-lessons.md b/tools/auto-improve-skill-lessons.md new file mode 100644 index 0000000..9794b36 --- /dev/null +++ b/tools/auto-improve-skill-lessons.md @@ -0,0 +1,433 @@ +# Auto-improve-skill — Lessons learned + +This is a **living doc**. The auto-pilot reads it during Phase 4 (Diagnose + ++ Modify). Every pilot adds patterns it discovered to the relevant +section. Pilot N benefits from patterns surfaced in pilots 1..N-1; the +auto-pilot doesn't have to rediscover them from zero. + +**How to use this from the prompt:** Phase 4 reads this file before +choosing what to modify. Match the failure pattern you observe to a +recipe below; if no recipe matches, do the diagnosis from first +principles, then add a new entry here at the end of the run. + +--- + +## The load-bearing prior + +> **Rules about *absence* (a missing attribute, a missing branch, a +> missing focus replacement) are 5–10× harder for models than rules +> about *presence* (a literal token in the code).** + +Source: manual web-design-guidelines run + auto-pilot supabase pilot +(2026-05-08) — both surfaced this independently. Use it to categorize +every missed rule before deciding what to modify. + +| Rule pattern | Relative miss rate | What helps | +|---|---|---| +| Visible bad pattern (literal token in code) | low | Often catches itself; the rule wording is enough | +| Anti-pattern that "looks normal" (e.g., ` + +// GOOD: stays enabled. Spinner appears during the request. + +``` +```` + +**Empirical evidence:** manual web-design-guidelines run — the rules +that needed examples (submit-disabled, paste-blocking, missing +autoComplete, image priority hint) all closed their miss rates by 60-100% +after the example was added. + +### E. Rationale + bug-story + +**When to use:** state-machine violations, lifecycle bugs, +non-obvious-failure rules. + +**Recipe:** narrate the failure case inline with the rule. Example: + +```markdown +NEVER `disabled={!form.valid}` — the user types, then deletes a +character to fix a typo, the button flickers off, and the paste-fill +races with state. Tested users will assume the button is broken. +``` + +The narration gives the model a "why this rule matters" hook that pure +declarative rules don't provide. + +--- + +## Grader-reliability patterns (Phase-2 build-suite recipes) + +These are common ways graders go wrong on first build. Pre-tune your +graders to avoid them; if you see the failure mode at baseline, fix the +grader as iteration 1 (do not propose a skill change yet). + +### G1. Line tolerance ±5–8 (not ±0–3) + +LLM line-counting is unreliable. Models report violations 1-3 lines off +from the actual line in multi-line JSX/SQL/code. Use the `looseRange` +helper (default tolerance ±8): + +```javascript +{ id: 'rule-id', lines: looseRange(18), keywords: [/.../i] } +// Accepts lines 10-26. +``` + +`looseRange(N, tolerance)` is defined in `_grader-utils.mjs`. Prefer it +over hand-rolling `range(N-3, N+3)` — the default `±8` absorbs the +drift seen on the current model matrix (sonnet-4.6, gpt-5, gemini-2.5-pro). +If you swap in a smaller or older model (e.g., gpt-5-mini, gpt-4o-mini), +expect 6–15 line drift and pass `looseRange(N, 12)` or wider. + +### G2. Hyphen-tolerant keyword regex + +Models output "empty-state" when the rule says "empty state", or +"clickable-handler" when the rule says "clickable handler". Use the +`fuzzyKeyword` helper: + +```javascript +keywords: [fuzzyKeyword('empty state')] // matches "empty state" and "empty-state" +keywords: [fuzzyKeyword('aria label')] // matches "aria-label" and "aria label" +``` + +`fuzzyKeyword(phrase)` is defined in `_grader-utils.mjs`. It escapes +regex metacharacters and replaces internal whitespace with `[-\s]*`, +so callers don't have to hand-roll the regex. + +### G3. Per-finding-line keyword matching (not whole-text) + +Don't `keywords.some(re => re.test(fullText))` — that produces spurious +cross-matches when keyword X appears in a different rule's finding line. +Use `_grader-utils.mjs`'s built-in per-finding-line matcher (split +findings.txt by line, match within each line). + +### G4. Multiple keyword variants + +Models phrase the same concept several ways: + ++ "covering" / "does not cover" / "missing covering index" ++ "label" / "aria-label" / "labeled" ++ "hover" / "hover state" / "hover:bg-*" + +Use the `tolerantKeyword` helper for word-stem matching: + +```javascript +keywords: [tolerantKeyword('cover')] // matches "cover", "covering", "covered" +keywords: [tolerantKeyword('label')] // matches "label", "labeled", "labels" +``` + +For multiple distinct stems on the same rule, use an array — the grader +treats them as alternatives: + +```javascript +keywords: [tolerantKeyword('hover'), fuzzyKeyword('hover state')] +``` + +Both `tolerantKeyword` and `fuzzyKeyword` are defined in `_grader-utils.mjs`. + +### G5. Set-semantics for sibling/list assertions + +When the grader checks a list of items, sort and compare — the model +emits items in different orders. + +```javascript +const names = pdf.repo_siblings_in_cohort_names.split(' | '); +assert.deepEqual(names.sort(), ['docx', 'xlsx']); // not deepEqual to ordered array +``` + +### G6. Verbosity floor for terse models + +Gemini sometimes outputs 3-4 line responses. Don't grade strict-pass on +"all 5 violations found" — many gemini failures are *truncated output*, +not missed rules. Compute rule-coverage rate (sum-found / sum-expected) +as the load-bearing metric instead of binary pass. + +--- + +## Default seeded violation types per skill shape + +When the auto-pilot builds a case in Phase 2, seed at least one +violation from each category for the skill's shape. This ensures +coverage of the absence-vs-presence axis and exposes whether the skill +needs Pattern A (two-pass workflow), Pattern C (per-element +checklists), or something else. + +### code-reviewer + +Seed at least one of each: + +1. Visible token misuse (e.g., `
` for action) +2. Missing attribute (e.g., `` without `autoComplete`) +3. Missing branch / no-empty-state (e.g., `array.map()` with no fallback for `[]`) +4. Anti-pattern that "looks normal" (e.g., `disabled={!form.valid}`) +5. State-machine violation (e.g., submit timing, focus on error) + +### tool-use / mcp-driver + +Seed at least one of each: + +1. Reaches-for-fallback (model uses `curl`/`npm i` instead of the prescribed CLI) +2. Wrong tool flag (passes `--user` when the skill calls for `--principal`) +3. Missing required step (skips snapshot, skips re-snapshot after action) +4. Output not validated (returns trace.jsonl without checking required artifacts) + +### document-producer + +Seed at least one of each: + +1. Missing required field in output (e.g., `answer.json` has no `risk_flags` key) +2. Wrong format (e.g., `2025-01-15` when the skill says `Intl.DateTimeFormat`) +3. Edge-case input (e.g., empty input, very long input, pre-corrupted file) +4. Format-only-correct: output validates but is unusable (e.g., PDF renders blank) + +### code-patterns + +Seed at least one of each: + +1. Wrong convention applied (skill says use 2-space indent, output uses 4) +2. Pattern not applied at all (skill says use `useReducer`, output uses `useState`) +3. Incorrect composition (uses prescribed pattern but in the wrong order) + +--- + +## Failure modes / known anti-patterns to avoid (Phase-4 don'ts) + +### Don't manufacture problems + +If baseline rule-coverage is ≥ 0.95, *exit clean*. Do not propose +modifications to a skill that already works. The goal is upstream PR +quality, not modification volume. + +**Source:** auto-pilot pdf pilot — baseline 1.00, no modifications +proposed. Maintainers will lose trust in our PRs if we open them for +non-issues. + +### Don't make breaking changes + +All proposed modifications must be **additive**: new sections, new +examples, new checklists. Never: + ++ Delete an existing rule ++ Change the wording of an existing rule ++ Reorder existing sections ++ Remove URLs or references in the skill + +This keeps the diff vs upstream small and the PR low-risk. + +### Don't burn iteration 1 on the wrong problem + +When baseline scores low, *first* check: is the grader the problem? Look +at the actual `findings.txt` from failed trials. If models *did* identify +the violations but the grader scored them wrong (line numbers off, +keyword mismatch, format variant), fix the grader as iteration 0 (don't +count it against the 2-iteration budget). + +**Source:** auto-pilot supabase + agent-browser pilots — both spent +iteration 1 on grader fixes before reaching skill modification. + +### Don't add bash commands to skills aimed at small models + +When you propose adding `grep`, `npm install`, `bash -c`, or any other +shell command to a SKILL.md, small models will try to **execute** them +rather than read them as documentation. This consistently regresses +coverage. + +**Source:** auto-pilot next-upgrade pilot (2026-05-09). Iteration 1 +added a per-element grep checklist with bash commands. Coverage +dropped 0.83 → 0.69 because gpt-5-mini executed `npx next-upgrade` +(fabricated) and `grep` calls instead of reading the files. + +For absence-type rules where you want the agent to look for missing +attributes, prefer pure declarative wording (Recipe C per-element +checklist) or BAD/GOOD code examples (Recipe D), not shell commands. + +### Watch for CLI fabrication on "upgrade-style" skills + +When a skill's name suggests transformation work (`upgrade`, `migrate`, +`convert`, `init`), some models will hallucinate a CLI matching the +skill name (e.g., `npx next-upgrade`), run it, get an error, and write +the error message as findings. This is *distinct* from the +"reaches-for-fallback `curl`" failure (Recipe B): there, the model +substitutes a wrong-but-real tool; here, the model invents one. + +**Source:** auto-pilot next-upgrade pilot (2026-05-09). gpt-5-mini ran +`npx next-upgrade` (does not exist), then wrote the package-not-found +error to `findings.txt`. + +If you observe this pattern at baseline, the fix is in the skill's +opening lines: be explicit that the agent should *read the source files* +and *write findings*, not run any CLI. Bundle that with the +"Verify-tool-installed" pattern (Recipe B) when applicable. + +### Some upstream repos use non-canonical SKILL.md paths + +Default `WebFetch` URL is +`https://raw.githubusercontent.com///main/skills//SKILL.md`. +Some repos use different layouts: + ++ `expo/skills`: `plugins/expo/skills//SKILL.md` ++ some agent-skills repos: `/SKILL.md` at the repo root (no + `skills/` prefix) + +Phase 1 should retry with these variants if the canonical URL 404s +before classifying as `blocked-by-skill-shape`. + +--- + +## Run-record protocol + +Every pilot adds an entry to one of these tables when it discovers +something new. Format: + +```markdown +**[skill-name] (date):** what was new — link to commit. +``` + +### Patterns added by pilots + ++ **manual web-design-guidelines (2026-05-06):** Two-pass workflow + per-element + checklists + 5 BAD/GOOD examples. Lifted 4-case suite from 72% → 86%. ++ **auto-pilot supabase (2026-05-08):** Independently rediscovered two-pass + workflow. Added it to a SQL skill. 0.54 → 0.86. ++ **auto-pilot agent-browser (2026-05-08):** Found that grader was over-strict + for non-interactive ops. Demoted snapshot from required to evidence-only. + Also surfaced "Verify-tool-installed nudge" pattern. ++ **auto-pilot pdf (2026-05-08):** Validated "exit clean on already-good skill" + — no modifications proposed; baseline 1.00. ++ **auto-pilot firebase-hosting-basics (2026-05-09):** Applied Recipe A + E to a + config-conventions skill. 0.89 → 1.00. ++ **auto-pilot shadcn-ui (2026-05-09):** Applied Recipe A + D. Gemini's + wrong-location miss rate dropped from 100% → 0%. 0.82 → 0.89. ++ **auto-pilot next-upgrade (2026-05-09):** First documented regression. Adding + bash commands to SKILL.md caused gpt-5-mini to execute them rather than read + files. 0.83 → 0.76. Surfaced two new failure modes: "no bash for small models" + and "CLI fabrication on upgrade-style skills". Both now in § Failure modes. ++ **auto-pilot building-native-ui / native-data-fetching (2026-05-09):** Surfaced + non-canonical SKILL.md path (`plugins/expo/skills//SKILL.md`). Now noted in + § Failure modes. + +### Grader patterns added by pilots + ++ **manual web-design-guidelines (2026-05-06):** ±5-8 line tolerance, hyphen + regex, per-finding-line matching, keyword variants. ++ **auto-pilot supabase (2026-05-08):** "covering" / "does not cover" alternation + pattern. Confirmed ±3 → ±8 line widening is needed by default. ++ **auto-pilot batch-2 (2026-05-09):** gpt-5-mini drifts 6–15 lines vs + sonnet/gemini's 0–3. The default `±8` `looseRange` is calibrated for sonnet + + gemini-2.5-pro + gpt-5 (the current matrix as of 2026-05-09) but undertuned + for smaller / older models. If swapping in a small model for cost reasons, + widen to `looseRange(N, 12)` as the new default. + +### Model-matrix history + ++ **2026-05-08:** `claude-sonnet-4.6`, `openai/gpt-5-mini`, `google/gemini-2.5-pro`. ++ **2026-05-09:** swapped `gpt-5-mini` → `gpt-5` after batch 2 showed gpt-5-mini + consistently dragged scores via (a) 3–4 line verbosity floor, (b) 6–15 line + drift in `findings.txt`, (c) CLI fabrication on transformation-style skills. + Final matrix: `claude-sonnet-4.6`, `openai/gpt-5`, `google/gemini-2.5-pro`. + +(Future pilots: append your additions here.) diff --git a/tools/auto-improve-skill-prompt.md b/tools/auto-improve-skill-prompt.md new file mode 100644 index 0000000..a458fa3 --- /dev/null +++ b/tools/auto-improve-skill-prompt.md @@ -0,0 +1,422 @@ +# Auto-improve a public agent skill + +You are running an autonomous skill-improvement pilot. Do all five +phases below without asking questions mid-run. If you can't proceed, +exit cleanly to `analysis.md` (see "Stop conditions" at the end). + +**Target slug:** `${SLUG}` — format `//`. + +**Reference run (the manual baseline you must reproduce on +web-design-guidelines):** `examples/workbench/web-design-guidelines/`. +If `examples/workbench/web-design-guidelines/` has source files +(`suite.yml`, `checks/`, etc.), read them as a layout reference. +If only `.results/` is present, the case sources are on a different +branch — proceed without it; the prompt is self-sufficient. + +--- + +## Setup + +1. Parse `${SLUG}` into `OWNER`, `REPO`, `SKILL_ID`. The case dir is + `examples/workbench/${SKILL_ID}/` — skill-id leaf only. The wrapper + has already created the empty dir for you. +2. Verify `OPENROUTER_API_KEY` is set; if not, exit with `analysis.md + status: blocked-by-error` and message "OPENROUTER_API_KEY not set". + +--- + +## Phase 1 — Discover + +1. Fetch the upstream `SKILL.md` via WebFetch from + `https://raw.githubusercontent.com///main/skills//SKILL.md` + (try `master` if `main` 404s; some repos use `/SKILL.md` + at the repo root — fall back to that if needed). +2. If the SKILL.md references one or more rules-doc URLs (look for + WebFetch instructions or raw GitHub URLs), fetch each. +3. Read the SKILL.md and any rules docs. Classify the skill type as + exactly one of: + - **document-producer** — produces structured output files (PDF, + docx, xlsx, JSON). Eval shape: graders inspect the produced file. + - **code-reviewer** — reads code and outputs findings. Eval shape: + seed code with known violations, grade on found findings. + - **tool-use / mcp-driver** — drives external tools or APIs. Eval + shape: graders inspect tool-call traces. + - **code-patterns** — prescribes code conventions / scaffolds. Eval + shape: ask agent to apply patterns to a starter, grade resulting + code. + - **other / unclear** — exit `analysis.md status: blocked-by-skill-shape`. +4. Pick the closest matching template under `examples/workbench/`: + - code-reviewer → use the case directory layout described in this + prompt (the structure under Phase 2). If + `examples/workbench/web-design-guidelines/` source files are + available, you can use them as a concrete example, but do not + require them. + - document-producer → mirror `examples/workbench/pdf/` + - tool-use / mcp-driver → mirror `examples/workbench/mcp/` + - code-patterns → use the layout described in Phase 2 (no + guaranteed local template for this type) +5. Persist the classification to `examples/workbench/${SKILL_ID}/analysis.md` + immediately (frontmatter only, status pending) so a partial run + leaves a trail. + +**Self-checkpoint:** if you can't classify with high confidence, exit +blocked. Do not invent a new shape. + +--- + +## Phase 2 — Build suite + +1. Write `examples/workbench/${SKILL_ID}/references/${SKILL_ID}/SKILL.md` + — a copy of the upstream skill, with one minimal tweak: change any + remote `WebFetch` calls in the skill to read from a local + `command.md` (or equivalent) bundled in the same `references/` + directory. This is for eval determinism. +2. If there's a remote rules doc, vendor it as + `references/${SKILL_ID}/`. +3. Seed sample input files in `workspace/`: 1–3 files matching the + skill's shape, 4–6 known violations per file, each violation on a + distinct line range mapped to one upstream rule. +4. Write graders in `checks/`: one `grade--findings.mjs` per + sample, sharing `checks/_grader-utils.mjs`. Write the following + file content to `examples/workbench/${SKILL_ID}/checks/_grader-utils.mjs` + (verbatim): + + ```js + // Shared grader logic for web-design-guidelines eval cases. + // + // Each finding is assumed to be one line in findings.txt that references + // ".tsx:" (line numbers come from the agent — they're often + // off by ±1-2 due to LLM line-counting). A violation is considered "found" + // when at least one finding line: + // (a) references a line number within the violation's accepted range, AND + // (b) contains at least one of the violation's distinguishing keywords. + // + // This per-finding-line check prevents spurious cross-matches (e.g. the + // keyword "label" from a different finding being credited to a paste rule). + + import { existsSync, readFileSync } from 'node:fs'; + + export function gradeFindings({ findingsPath, file, expected }) { + const failures = []; + const found = new Set(); + + if (!existsSync(findingsPath)) { + failures.push('findings.txt was not created'); + return emitResult({ found, expected, failures }); + } + + const text = readFileSync(findingsPath, 'utf-8'); + const refRe = new RegExp(`${escapeRe(file)}\\s*[:#]\\s*(\\d+)`, 'i'); + const findingLines = text.split(/\r?\n/).filter((ln) => refRe.test(ln)); + + for (const v of expected) { + for (const line of findingLines) { + const m = line.match(refRe); + if (!m) continue; + const lineNum = Number(m[1]); + if (!v.lines.includes(lineNum)) continue; + if (!v.keywords.some((re) => re.test(line))) continue; + found.add(v.id); + break; + } + } + + return emitResult({ found, expected, failures }); + } + + function emitResult({ found, expected, failures }) { + const missing = expected.filter((v) => !found.has(v.id)).map((v) => v.id); + const score = found.size / expected.length; + const pass = found.size === expected.length; + + console.log(JSON.stringify({ + pass, + score, + evidence: [ + `${found.size}/${expected.length} expected violations identified`, + ...[...found].map((id) => `+ ${id}`), + ...missing.map((id) => `- missing: ${id}`), + ...failures, + ], + })); + return pass; + } + + function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + // Helper: build an inclusive line range [start, start+1, ..., end]. + export function range(start, end) { + const out = []; + for (let i = start; i <= end; i++) out.push(i); + return out; + } + + // Helper: centered loose range — accepts the violation line ± tolerance. + // Default tolerance ±8 handles LLM line-counting drift on multi-line elements. + // PREFER this over `range(N-3, N+3)` — see lessons.md § G1. + export function looseRange(centerLine, tolerance = 8) { + return range(centerLine - tolerance, centerLine + tolerance); + } + + // Helper: hyphen-tolerant keyword regex — `fuzzyKeyword('empty state')` + // matches both "empty state" and "empty-state" and "emptystate". + // PREFER this over hand-writing `/empty[-\s]+state/` — see lessons.md § G2. + export function fuzzyKeyword(phrase) { + const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flexible = escaped.replace(/\s+/g, '[-\\s]*'); + return new RegExp(flexible, 'i'); + } + + // Helper: prefix-tolerant keyword — `tolerantKeyword('cover')` matches + // "cover", "covering", "covered", "does not cover". + // PREFER this over `/covering/i` — see lessons.md § G4. + export function tolerantKeyword(stem) { + const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`\\b${escaped}\\w*`, 'i'); + } + ``` + +5. Write `suite.yml` with the standard 3-model matrix: + + ```yaml + models: + - openrouter/anthropic/claude-sonnet-4.6 + - openrouter/openai/gpt-5 + - openrouter/google/gemini-2.5-pro + env: + - OPENROUTER_API_KEY + timeoutSeconds: 600 + ``` + +6. Write a Cases-table README. If + `examples/workbench/web-design-guidelines/README.md` is available, + use it as a concrete example; otherwise use the following skeleton + (fill in all `<…>` placeholders): + + ```markdown + # eval + + Eval suite for + [`//`](https://github.com//) — + . + + ## Cases + + ### `review-` — + + Sample: `workspace/.` + + | Line | Violation | Rule | + |---|---|---| + | | | | + | | | | + + (repeat for each seeded violation) + + ## Vendored snapshot + + The skill normally . For deterministic eval + we vendor a snapshot at `references//` and + tweak `SKILL.md` to read it locally. Diff vs upstream is one line. + + ## Run + + \`\`\`bash + export OPENROUTER_API_KEY=sk-or-... + npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 + \`\`\` + + ## Models + + The suite runs a 3-provider mid-tier matrix: + + - `openrouter/anthropic/claude-sonnet-4.6` + - `openrouter/openai/gpt-5` + - `openrouter/google/gemini-2.5-pro` + ``` + +**Self-checkpoint:** if you can't seed ≥3 reasonable violations, exit +`status: blocked-by-skill-shape`. + +--- + +## Phase 3 — Baseline + +1. From the case directory, run: + + ```bash + set -a; . ./.env; set +a + npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 \ + 2>&1 | tee /tmp/${SKILL_ID}-baseline.log + ``` + +2. Aggregate per-rule miss frequency, per-model strict-pass rate, and + overall **rule-coverage rate** = (sum identified) / (sum expected) + across all trials. +3. If baseline rule-coverage < 0.50, exit `status: blocked-by-skill-shape`. +4. If baseline rule-coverage ≥ 0.95, exit `status: success` with + `final_rule_coverage = baseline` (skill needs no changes on these + cases). +5. Persist baseline numbers in `analysis.md`. + +--- + +## Phase 4 — Iterate (max 2 loops) + +**Before iterating, read `tools/auto-improve-skill-lessons.md`** — it has +recipes A-E for optimization patterns and G1-G6 for grader-reliability +patterns, each with empirical evidence from prior pilots. Match your +observed failure pattern to a recipe before designing a custom fix. + +For each iteration `I` (1 then 2): + +1. **Diagnose** — list the highest-miss-frequency rules. Use this prior + (from `auto-improve-skill-lessons.md` § "The load-bearing prior"): + + > Rules about *absence* (a missing attribute, branch, or focus + > replacement) are 5–10× harder than rules about *presence* (a + > literal token in code). Examples and per-element checklists help + > most for absence-type rules. + + Categorize each missed rule: visible-pattern / absence-of-attribute / + state-machine / subjective. + + **Grader-vs-skill check (do this first):** look at actual + `findings.txt` from failed trials. If models *did* identify the + violations but the grader scored them wrong (line numbers off, + keyword mismatch, format variant), the grader is the bug. Apply + recipes G1-G6 from the lessons doc and re-run *without* counting + this against the 2-iteration budget. Only proceed to skill + modification once the grader is calibrated. + +2. **Modify** — write a *minimal additive* edit using the recipes + from `auto-improve-skill-lessons.md` § "Optimization patterns": + - **Recipe A** (two-pass workflow) for code-reviewer skills with + mixed presence/absence rules + - **Recipe B** (verify-tool-installed nudge) for tool-use skills + where models fall back to `curl`/`npm i` + - **Recipe C** (per-element checklists) for skills with rules + grouped by element type + - **Recipe D** (BAD/GOOD examples) for anti-patterns where the + bad pattern looks idiomatic + - **Recipe E** (rationale + bug-story) for state-machine + violations + + Edits must be additive: no rule deletions, no wording changes to + existing rules. (See lessons doc § "Don't make breaking changes".) + + After the run completes, **append a one-line entry to + `tools/auto-improve-skill-lessons.md` § "Run-record protocol"** + documenting any new pattern your pilot surfaced. The doc is a + living artifact; future pilots benefit from yours. + +3. **Re-run** the same `run-suite --trials 3` command and compute new + rule-coverage. + +4. **Decide:** + - `new - baseline ≥ +0.05` → stop, success. + - `I == 2` → stop, uplift-too-small. + - Else loop. + +**Cost guard:** sum `metrics.cost.total` from each run's `result.json`. +If cumulative cost > $7.00, exit `status: budget-exceeded` immediately. +Leave a $2-3 buffer below the wrapper's `--max-budget-usd` so you have +room to finish "Always: write analysis.md AND commit" cleanly. + +--- + +## Phase 5 — Package + +If final status is `success`: + +1. Create `proposed-upstream-changes/` with: + + ```text + proposed-upstream-changes/ + README.md + /before-SKILL.md + /after-SKILL.md + /before-.md # if separate rules doc + /after-.md + ``` + +2. The `after-SKILL.md` must contain the proposed upstream change but + NOT the local-path tweak from Phase 2 (revert that line). Diff vs + upstream should be purely additive. + +3. Write `proposed-upstream-changes/README.md`. If + `examples/workbench/web-design-guidelines/proposed-upstream-changes/README.md` + is available, use it as a style reference; otherwise write a short + summary covering: what changed, why (evidence from eval results), + and how to apply the diff upstream. + +If status is anything else, skip Phase 5. + +--- + +## Always: write `analysis.md` AND commit (do NOT push) + +**This is a single atomic step. Both must happen, in this order, every time +the run ends — success, blocked, or out of budget.** Do not write +`analysis.md` and stop there. Do not commit without writing +`analysis.md` first. + +Step A — write `examples/workbench/${SKILL_ID}/analysis.md`: + +```markdown +--- +skill: ${SLUG} +status: success | uplift-too-small | blocked-by-skill-shape | + budget-exceeded | blocked-by-error +classification: code-reviewer | document-producer | tool-use | code-patterns +baseline_rule_coverage: 0.NN +final_rule_coverage: 0.NN +modifications_tried: N +total_cost_usd: NN.NN +--- + +# Auto-pilot run for `${SLUG}` + +3–6 short bullets covering: classification rationale, what you seeded, +baseline failure pattern, modification tried + reason, uplift result, +any judgment calls. +``` + +Step B — IMMEDIATELY after writing `analysis.md`, run these git commands +(do not pause, do not call any other tool first, just run them): + +```bash +git checkout -b eval/auto-pilot/${SKILL_ID} +git add examples/workbench/${SKILL_ID}/suite.yml \ + examples/workbench/${SKILL_ID}/README.md \ + examples/workbench/${SKILL_ID}/analysis.md \ + examples/workbench/${SKILL_ID}/references/ \ + examples/workbench/${SKILL_ID}/workspace/ \ + examples/workbench/${SKILL_ID}/checks/ +[ -d examples/workbench/${SKILL_ID}/proposed-upstream-changes ] \ + && git add examples/workbench/${SKILL_ID}/proposed-upstream-changes/ +git commit -m "eval(auto-pilot): ${SKILL_ID} — status=, coverage " +``` + +Do **not** `git push`. The orchestrator reads `analysis.md` and reports. + +If you find yourself running low on budget or context: **skip everything +else and do this section first.** A run with results-but-no-analysis-or-commit +is worse than a run with truncated results that committed cleanly. + +--- + +## Stop conditions (summary) + +| Condition | Action | +| --- | --- | +| Two iterations of Phase 4 done | Stop, write `analysis.md` | +| Cumulative cost > $7.00 (or 70% of wrapper budget) | Stop, `status: budget-exceeded` | +| Phase 1 can't classify | Stop, `status: blocked-by-skill-shape` | +| Phase 2 can't seed ≥3 violations | Stop, `status: blocked-by-skill-shape` | +| Baseline rule-coverage < 0.50 | Stop, `status: blocked-by-skill-shape` | +| Hard error not recovered in 1 retry | Stop, `status: blocked-by-error` | + +You **never** ask the operator a question mid-run. diff --git a/tools/auto-improve-skill.mjs b/tools/auto-improve-skill.mjs new file mode 100644 index 0000000..93216a6 --- /dev/null +++ b/tools/auto-improve-skill.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +// Auto-improve-skill wrapper. +// +// Operator usage (from this Claude Code session via Bash): +// node tools/auto-improve-skill.mjs // [--force] [--budget ] +// +// Flags: +// --force overwrite an existing examples/workbench// +// --budget per-run claude -p budget cap (default: 10.00) +// The prompt's Phase-4 cost guard stops the agent at $7 +// so the last $3 covers analysis.md + commit cleanup. +// +// Spawns `claude -p` with the templated prompt; the inner agent does the +// 5-phase work (vendor → build suite → baseline → iterate → package) and +// writes results under examples/workbench//. + +import { spawn } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, createWriteStream } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..'); +const PROMPT_PATH = join(HERE, 'auto-improve-skill-prompt.md'); +const PER_CALL_TIMEOUT_MS = 90 * 60 * 1000; // 90 min hard wall-clock cap + +const args = process.argv.slice(2); +const FORCE = args.includes('--force'); + +function parseBudgetFlag() { + const i = args.indexOf('--budget'); + if (i < 0 || !args[i + 1]) return '10.00'; + const v = args[i + 1]; + if (!/^\d+(\.\d+)?$/.test(v) || Number(v) <= 0) { + console.error(`bad --budget: "${v}" — expected a positive number`); + process.exit(2); + } + return v; +} +const BUDGET = parseBudgetFlag(); +const BUDGET_FLAG_IDX = args.indexOf('--budget'); +const BUDGET_VALUE_IDX = BUDGET_FLAG_IDX < 0 ? null : BUDGET_FLAG_IDX + 1; + +const slug = args.find((a, i) => !a.startsWith('--') && i !== BUDGET_VALUE_IDX); +if (!slug) { + console.error('usage: auto-improve-skill.mjs // [--force] [--budget ]'); + process.exit(2); +} +const parts = slug.split('/'); +if (parts.length !== 3) { + console.error(`bad slug: "${slug}" — expected //`); + process.exit(2); +} +const [, , skillId] = parts; +const caseDir = join(REPO_ROOT, 'examples/workbench', skillId); + +if (existsSync(caseDir) && !FORCE) { + console.error(`refusing: ${caseDir} already exists. Pass --force to overwrite.`); + process.exit(2); +} +mkdirSync(caseDir, { recursive: true }); + +const promptTemplate = readFileSync(PROMPT_PATH, 'utf-8'); +const prompt = promptTemplate.replace(/\$\{SLUG\}/g, slug).replace(/\$\{SKILL_ID\}/g, skillId); + +const logPath = join(caseDir, '.run.log'); +const logStream = createWriteStream(logPath, { flags: 'a' }); +console.log(`spawning claude -p for ${slug} → ${caseDir} (log: ${logPath})`); + +const claudeArgs = [ + '-p', prompt, + '--model', 'sonnet', + '--no-session-persistence', + '--disable-slash-commands', + '--dangerously-skip-permissions', + '--max-budget-usd', BUDGET, +]; +const childEnv = { ...process.env }; +delete childEnv.CLAUDECODE; + +const child = spawn('claude', claudeArgs, { + cwd: REPO_ROOT, + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'], +}); + +let timedOut = false; +const timer = setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + setTimeout(() => child.kill('SIGKILL'), 5000).unref(); +}, PER_CALL_TIMEOUT_MS); + +child.stdout.on('data', (chunk) => { process.stdout.write(chunk); logStream.write(chunk); }); +child.stderr.on('data', (chunk) => { process.stderr.write(chunk); logStream.write(chunk); }); +child.on('close', (code) => { + clearTimeout(timer); + logStream.end(); + if (timedOut) { + console.error(`\n[wrapper] claude -p exceeded ${PER_CALL_TIMEOUT_MS / 60000}-min timeout`); + process.exit(124); + } + const analysisPath = join(caseDir, 'analysis.md'); + if (existsSync(analysisPath)) { + console.log(`\n[wrapper] analysis.md: ${analysisPath}`); + } else { + console.error(`\n[wrapper] no analysis.md was written; check ${logPath}`); + } + process.exit(code ?? 1); +});